chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
agapple
2020-02-28 13:15:27 +08:00
parent 18418ebd98
commit 5e761e14fa
47 changed files with 1378 additions and 1290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/ES6xAdapter.java
View File

@@ -5,6 +5,10 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import javax.sql.DataSource;
import org.elasticsearch.action.search.SearchResponse;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es6x.etl.ESEtlService;
@@ -14,9 +18,6 @@ import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import org.elasticsearch.action.search.SearchResponse;
import javax.sql.DataSource;
/**
* ES 6.x 外部适配器

224
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/etl/ESEtlService.java
View File

@@ -10,7 +10,6 @@ import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
@@ -79,119 +78,118 @@ public class ESEtlService extends AbstractEtlService {
}
// 如果是主键字段则不插入
if (fieldItem.getFieldName().equals(mapping.get_id())) {
idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
} else {
Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
esFieldData.put(Util.cleanColumn(fieldName), val);
}
}
if (!mapping.getRelations().isEmpty()) {
mapping.getRelations().forEach((relationField, relationMapping) -> {
Map<String, Object> relations = new HashMap<>();
relations.put("name", relationMapping.getName());
if (StringUtils.isNotEmpty(relationMapping.getParent())) {
FieldItem parentFieldItem = mapping.getSchemaItem()
.getSelectFields()
.get(relationMapping.getParent());
Object parentVal;
try {
parentVal = esTemplate.getValFromRS(mapping,
rs,
parentFieldItem.getFieldName(),
parentFieldItem.getFieldName());
} catch (SQLException e) {
throw new RuntimeException(e);
}
if (parentVal != null) {
relations.put("parent", parentVal.toString());
esFieldData.put("$parent_routing", parentVal.toString());
}
}
esFieldData.put(Util.cleanColumn(relationField), relations);
});
}
if (idVal != null) {
String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(
mapping.get_index(),
mapping.get_type(),
idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) {
esUpdateRequest.setRouting(parentVal);
}
esBulkRequest.add(esUpdateRequest);
} else {
ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping
.get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) {
esIndexRequest.setRouting(parentVal);
}
esBulkRequest.add(esIndexRequest);
}
} else {
idVal = esFieldData.get(mapping.getPk());
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
.size(10000);
SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping
.get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
esBulkRequest.add(esUpdateRequest);
}
}
if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
&& esBulkRequest.numberOfActions() > 0) {
long esBatchBegin = System.currentTimeMillis();
ESBulkResponse rp = esBulkRequest.bulk();
if (rp.hasFailures()) {
rp.processFailBulkResponse("全量数据 etl 异常 ");
}
if (logger.isTraceEnabled()) {
logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
(System.currentTimeMillis() - batchBegin),
(System.currentTimeMillis() - esBatchBegin),
esBulkRequest.numberOfActions(),
mapping.get_index());
}
batchBegin = System.currentTimeMillis();
esBulkRequest.resetBulk();
}
count++;
impCount.incrementAndGet();
}
if (esBulkRequest.numberOfActions() > 0) {
long esBatchBegin = System.currentTimeMillis();
ESBulkResponse rp = esBulkRequest.bulk();
if (rp.hasFailures()) {
rp.processFailBulkResponse("全量数据 etl 异常 ");
}
if (logger.isTraceEnabled()) {
logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
(System.currentTimeMillis() - batchBegin),
(System.currentTimeMillis() - esBatchBegin),
esBulkRequest.numberOfActions(),
mapping.get_index());
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
throw new RuntimeException(e);
if (fieldItem.getFieldName().equals(mapping.get_id())) {
idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
} else {
Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
esFieldData.put(Util.cleanColumn(fieldName), val);
}
return count;
});
}
if (!mapping.getRelations().isEmpty()) {
mapping.getRelations().forEach((relationField, relationMapping) -> {
Map<String, Object> relations = new HashMap<>();
relations.put("name", relationMapping.getName());
if (StringUtils.isNotEmpty(relationMapping.getParent())) {
FieldItem parentFieldItem = mapping.getSchemaItem()
.getSelectFields()
.get(relationMapping.getParent());
Object parentVal;
try {
parentVal = esTemplate.getValFromRS(mapping,
rs,
parentFieldItem.getFieldName(),
parentFieldItem.getFieldName());
} catch (SQLException e) {
throw new RuntimeException(e);
}
if (parentVal != null) {
relations.put("parent", parentVal.toString());
esFieldData.put("$parent_routing", parentVal.toString());
}
}
esFieldData.put(Util.cleanColumn(relationField), relations);
});
}
if (idVal != null) {
String parentVal = (String) esFieldData.remove("$parent_routing");
if (mapping.isUpsert()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
if (StringUtils.isNotEmpty(parentVal)) {
esUpdateRequest.setRouting(parentVal);
}
esBulkRequest.add(esUpdateRequest);
} else {
ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(),
mapping.get_type(),
idVal.toString()).setSource(esFieldData);
if (StringUtils.isNotEmpty(parentVal)) {
esIndexRequest.setRouting(parentVal);
}
esBulkRequest.add(esIndexRequest);
}
} else {
idVal = esFieldData.get(mapping.getPk());
ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
SearchResponse response = esSearchRequest.getResponse();
for (SearchHit hit : response.getHits()) {
ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(),
mapping.get_type(),
hit.getId()).setDoc(esFieldData);
esBulkRequest.add(esUpdateRequest);
}
}
if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 && esBulkRequest.numberOfActions() > 0) {
long esBatchBegin = System.currentTimeMillis();
ESBulkResponse rp = esBulkRequest.bulk();
if (rp.hasFailures()) {
rp.processFailBulkResponse("全量数据 etl 异常 ");
}
if (logger.isTraceEnabled()) {
logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
(System.currentTimeMillis() - batchBegin),
(System.currentTimeMillis() - esBatchBegin),
esBulkRequest.numberOfActions(),
mapping.get_index());
}
batchBegin = System.currentTimeMillis();
esBulkRequest.resetBulk();
}
count++;
impCount.incrementAndGet();
}
if (esBulkRequest.numberOfActions() > 0) {
long esBatchBegin = System.currentTimeMillis();
ESBulkResponse rp = esBulkRequest.bulk();
if (rp.hasFailures()) {
rp.processFailBulkResponse("全量数据 etl 异常 ");
}
if (logger.isTraceEnabled()) {
logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
(System.currentTimeMillis() - batchBegin),
(System.currentTimeMillis() - esBatchBegin),
esBulkRequest.numberOfActions(),
mapping.get_index());
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
throw new RuntimeException(e);
}
return count;
} );
return true;
} catch (Exception e) {

8
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ES6xTemplate.java
View File

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es6x.support;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -20,8 +24,8 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESDeleteRequest;

19
client-adapter/es6x/src/main/java/com/alibaba/otter/canal/client/adapter/es6x/support/ESConnection.java
View File

@@ -5,7 +5,6 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Map;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
@@ -27,7 +26,11 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.*;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.RestHighLevelClientExt;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -40,6 +43,8 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest;
/**
* ES 连接器, Transport Rest 两种方式
*
@@ -51,7 +56,7 @@ public class ESConnection {
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
public enum ESClientMode {
TRANSPORT, REST
TRANSPORT, REST
}
private ESClientMode mode;
@@ -86,10 +91,9 @@ public class ESConnection {
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
String[] nameAndPwdArr = nameAndPwd.split(":");
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
restHighLevelClient = new RestHighLevelClient(restClientBuilder);
}
@@ -418,6 +422,7 @@ public class ESConnection {
}
}
@SuppressWarnings("deprecation")
public ESBulkResponse bulk() {
if (mode == ESClientMode.TRANSPORT) {
BulkResponse responses = bulkRequestBuilder.execute().actionGet();

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSub2Test.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/LabelSyncJoinSubTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOne2Test.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/RoleSyncJoinOneTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncJoinOneTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;

6
client-adapter/es6x/src/test/java/com/alibaba/otter/canal/client/adapter/es6x/test/sync/UserSyncSingleTest.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.es6x.test.sync;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.elasticsearch.action.get.GetResponse;
import org.junit.Assert;

8
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/ES7xAdapter.java
View File

@@ -5,20 +5,20 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import javax.sql.DataSource;
import org.elasticsearch.action.search.SearchResponse;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es7x.etl.ESEtlService;
import com.alibaba.otter.canal.client.adapter.es7x.support.ES7xTemplate;
import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import javax.sql.DataSource;
/**
* ES 7.x 外部适配器
*

6
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ES7xTemplate.java
View File

@@ -2,7 +2,11 @@ package com.alibaba.otter.canal.client.adapter.es7x.support;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

10
client-adapter/es7x/src/main/java/com/alibaba/otter/canal/client/adapter/es7x/support/ESConnection.java
View File

@@ -54,11 +54,12 @@ public class ESConnection {
private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
public enum ESClientMode {
TRANSPORT, REST
TRANSPORT, REST
}
private ESClientMode mode;
@SuppressWarnings("deprecation")
private TransportClient transportClient;
private RestHighLevelClient restHighLevelClient;
@@ -89,10 +90,9 @@ public class ESConnection {
if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
String[] nameAndPwdArr = nameAndPwd.split(":");
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(
httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nameAndPwdArr[0],
nameAndPwdArr[1]));
restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
restHighLevelClient = new RestHighLevelClient(restClientBuilder);
}

6
client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ES7xTest.java
View File

@@ -24,6 +24,7 @@ import org.junit.Test;
@Ignore
public class ES7xTest {
@SuppressWarnings("deprecation")
private TransportClient transportClient;
@Before
@@ -65,8 +66,9 @@ public class ES7xTest {
relations.put("parent", "2");
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
bulkRequestBuilder
.add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData));
bulkRequestBuilder.add(transportClient.prepareIndex("test", "osm", "2_4")
.setRouting("2")
.setSource(esFieldData));
commit(bulkRequestBuilder);
}

2
client-adapter/es7x/src/test/java/com/alibaba/otter/canal/client/adapter/es7x/test/ESConnectionTest.java
View File

@@ -8,9 +8,9 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection;
import org.springframework.util.Assert;
@Ignore
public class ESConnectionTest {

57
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.client.adapter.es.core;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -8,7 +7,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import org.apache.commons.lang.StringUtils;
import com.alibaba.druid.pool.DruidDataSource;
@@ -19,6 +17,7 @@ import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor;
import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
@@ -63,8 +62,8 @@ public abstract class ESAdapter implements OuterAdapter {
// 过滤不匹配的key的配置
esSyncConfigTmp.forEach((key, config) -> {
if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
|| (config.getOuterAdapterKey() != null
&& config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
|| (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
.equalsIgnoreCase(configuration.getKey()))) {
esSyncConfig.put(key, config);
}
});
@@ -104,12 +103,12 @@ public abstract class ESAdapter implements OuterAdapter {
String table = dml.getTable();
Map<String, ESSyncConfig> configMap;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
configMap = dbTableEsSyncConfig
.get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
+ "_" + database + "-" + table);
configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "-"
+ StringUtils.trimToEmpty(dml.getGroupId()) + "_" + database + "-"
+ table);
} else {
configMap = dbTableEsSyncConfig
.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-"
+ table);
}
if (configMap != null && !configMap.values().isEmpty()) {
@@ -155,23 +154,29 @@ public abstract class ESAdapter implements OuterAdapter {
}
String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
Map<String, ESSyncConfig> esSyncConfigMap;
if (envProperties != null
&& !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
esSyncConfigMap = dbTableEsSyncConfig
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
+ StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-"
+ tableItem.getTableName(),
k -> new ConcurrentHashMap<>());
} else {
esSyncConfigMap = dbTableEsSyncConfig
.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-"
+ tableItem.getTableName(),
k -> new ConcurrentHashMap<>());
}
schemaItem.getAliasTableItems()
.values()
.forEach(tableItem -> {
Map<String, ESSyncConfig> esSyncConfigMap;
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+ "-"
+ StringUtils.trimToEmpty(config.getGroupId())
+ "_"
+ schema
+ "-"
+ tableItem.getTableName(),
k -> new ConcurrentHashMap<>());
} else {
esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+ "_"
+ schema
+ "-"
+ tableItem.getTableName(),
k -> new ConcurrentHashMap<>());
}
esSyncConfigMap.put(configName, config);
});
esSyncConfigMap.put(configName, config);
});
}
}

15
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java
View File

@@ -9,8 +9,19 @@ import java.util.stream.Collectors;
import com.alibaba.fastsql.sql.SQLUtils;
import com.alibaba.fastsql.sql.ast.SQLExpr;
import com.alibaba.fastsql.sql.ast.expr.*;
import com.alibaba.fastsql.sql.ast.statement.*;
import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLCaseExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLCharExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr;
import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr;
import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
import com.alibaba.fastsql.sql.ast.statement.SQLJoinTableSource;
import com.alibaba.fastsql.sql.ast.statement.SQLSelectGroupByClause;
import com.alibaba.fastsql.sql.ast.statement.SQLSelectItem;
import com.alibaba.fastsql.sql.ast.statement.SQLSelectStatement;
import com.alibaba.fastsql.sql.ast.statement.SQLSubqueryTableSource;
import com.alibaba.fastsql.sql.ast.statement.SQLTableSource;
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.fastsql.sql.parser.ParserException;

27
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java
View File

@@ -1,13 +1,9 @@
package com.alibaba.otter.canal.client.adapter.es.core.monitor;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -15,12 +11,9 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.Util;
@@ -70,8 +63,11 @@ public class ESConfigMonitor {
try {
// 加载新增的配置文件
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
ESSyncConfig config = YmlConfigBinder
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
configContent,
ESSyncConfig.class,
null,
envProperties);
if (config != null) {
config.validate();
addConfigToCache(file, config);
@@ -89,14 +85,17 @@ public class ESConfigMonitor {
try {
if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
// 加载配置文件
String configContent = MappingConfigsLoader
.loadConfig(adapterName + File.separator + file.getName());
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
+ file.getName());
if (configContent == null) {
onFileDelete(file);
return;
}
ESSyncConfig config = YmlConfigBinder
.bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
configContent,
ESSyncConfig.class,
null,
envProperties);
if (config == null) {
return;
}

10
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java
View File

@@ -1,22 +1,26 @@
package com.alibaba.otter.canal.client.adapter.es.core.service;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem;
import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;

8
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java
View File

@@ -5,7 +5,13 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.sql.Blob;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.joda.time.DateTime;

280
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -5,19 +5,24 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.support.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.support.*;
import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.google.common.base.Joiner;
/**
@@ -79,7 +84,8 @@ public class HbaseEtlService extends AbstractEtlService {
createTable();
// 拼接sql
String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable() + "`";
String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable()
+ "`";
return super.importData(sql, params);
} catch (Exception e) {
@@ -120,146 +126,146 @@ public class HbaseEtlService extends AbstractEtlService {
if (rowKeyColumns != null) {
// 取rowKey字段拼接
StringBuilder rowKeyVale = new StringBuilder();
for (String rowKeyColumnName : rowKeyColumns) {
Object obj = rs.getObject(rowKeyColumnName);
if (obj != null) {
rowKeyVale.append(obj.toString());
}
rowKeyVale.append("|");
}
int len = rowKeyVale.length();
if (len > 0) {
rowKeyVale.delete(len - 1, len);
}
row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
StringBuilder rowKeyVale = new StringBuilder();
for (String rowKeyColumnName : rowKeyColumns) {
Object obj = rs.getObject(rowKeyColumnName);
if (obj != null) {
rowKeyVale.append(obj.toString());
}
rowKeyVale.append("|");
}
int len = rowKeyVale.length();
if (len > 0) {
rowKeyVale.delete(len - 1, len);
}
row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
}
for (int j = 1; j <= cc; j++) {
String columnName = rs.getMetaData().getColumnName(j);
Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
if (val == null) {
continue;
}
MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
// 没有配置映射
if (columnItem == null) {
String family = hbaseMapping.getFamily();
String qualifile = columnName;
if (hbaseMapping.isUppercaseQualifier()) {
qualifile = qualifile.toUpperCase();
}
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(Bytes.toBytes(val.toString()));
} else {
row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
}
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
Type type = Type.getType(classes[j - 1]);
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(TypeUtil.toBytes(val, type));
} else {
row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
}
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
PhType phType = PhType.getType(classes[j - 1]);
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else {
row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
}
}
} else {
// 如果不需要类型转换
if (columnItem.getType() == null || "".equals(columnItem.getType())) {
if (val instanceof java.sql.Date) {
SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
val = dateFmt.format((Date) val);
} else if (val instanceof Timestamp) {
SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
val = datetimeFmt.format((Date) val);
}
for (int j = 1; j <= cc; j++) {
String columnName = rs.getMetaData().getColumnName(j);
Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
if (val == null) {
continue;
byte[] valBytes = Bytes.toBytes(val.toString());
if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
row.setRowKey(valBytes);
} else {
row.setRowKey(valBytes);
}
MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
// 没有配置映射
if (columnItem == null) {
String family = hbaseMapping.getFamily();
String qualifile = columnName;
if (hbaseMapping.isUppercaseQualifier()) {
qualifile = qualifile.toUpperCase();
} else {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
}
} else {
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
byte[] valBytes = Bytes.toBytes(val.toString());
if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
}
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(Bytes.toBytes(val.toString()));
} else {
row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
}
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
Type type = Type.getType(classes[j - 1]);
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(TypeUtil.toBytes(val, type));
} else {
row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
}
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
PhType phType = PhType.getType(classes[j - 1]);
if (hbaseMapping.getRowKey() == null && j == 1) {
row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else {
row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
}
row.setRowKey(valBytes);
} else {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
}
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
Type type = Type.getType(columnItem.getType());
if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
String v = limitLenNum(columnItem.getRowKeyLen(), val);
row.setRowKey(Bytes.toBytes(v));
} else {
row.setRowKey(TypeUtil.toBytes(val, type));
}
} else {
// 如果不需要类型转换
if (columnItem.getType() == null || "".equals(columnItem.getType())) {
if (val instanceof java.sql.Date) {
SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
val = dateFmt.format((Date) val);
} else if (val instanceof Timestamp) {
SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
val = datetimeFmt.format((Date) val);
}
byte[] valBytes = Bytes.toBytes(val.toString());
if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
row.setRowKey(valBytes);
} else {
row.setRowKey(valBytes);
}
} else {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
}
} else {
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
byte[] valBytes = Bytes.toBytes(val.toString());
if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
}
row.setRowKey(valBytes);
} else {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
}
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
Type type = Type.getType(columnItem.getType());
if (columnItem.isRowKey()) {
if (columnItem.getRowKeyLen() != null) {
String v = limitLenNum(columnItem.getRowKeyLen(), val);
row.setRowKey(Bytes.toBytes(v));
} else {
row.setRowKey(TypeUtil.toBytes(val, type));
}
} else {
row.addCell(columnItem.getFamily(),
columnItem.getQualifier(),
TypeUtil.toBytes(val, type));
}
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
PhType phType = PhType.getType(columnItem.getType());
if (columnItem.isRowKey()) {
row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else {
row.addCell(columnItem.getFamily(),
columnItem.getQualifier(),
PhTypeUtil.toBytes(val, phType));
}
}
}
row.addCell(columnItem.getFamily(),
columnItem.getQualifier(),
TypeUtil.toBytes(val, type));
}
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
PhType phType = PhType.getType(columnItem.getType());
if (columnItem.isRowKey()) {
row.setRowKey(PhTypeUtil.toBytes(val, phType));
} else {
row.addCell(columnItem.getFamily(),
columnItem.getQualifier(),
PhTypeUtil.toBytes(val, phType));
}
}
if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
rows.add(row);
complete = false;
if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rows.clear();
complete = true;
}
i++;
impCount.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug("successful import count:" + impCount.get());
}
}
if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
}
} catch (Exception e) {
logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
// throw new RuntimeException(e);
}
return i;
});
}
if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
rows.add(row);
complete = false;
if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
rows.clear();
complete = true;
}
i++;
impCount.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug("successful import count:" + impCount.get());
}
}
if (!complete && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
}
} catch (Exception e) {
logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
// throw new RuntimeException(e);
}
return i;
} );
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);

4
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/AdapterConfigHolder.java
View File

@@ -1,10 +1,10 @@
package com.alibaba.otter.canal.adapter.launcher.config;
import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import com.alibaba.otter.canal.adapter.launcher.monitor.remote.ConfigItem;
/**
* @author L.J.R @ 2019-09-03
* @version 1.1.4

8
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java
View File

@@ -8,20 +8,20 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.connector.core.config.CanalConstants;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
/**
* 适配处理器

257
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -1,8 +1,5 @@
package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -33,13 +30,13 @@ import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
*/
public class CanalAdapterLoader {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private CanalClientConfig canalClientConfig;
private CanalClientConfig canalClientConfig;
private Map<String, AdapterProcessor> canalAdapterProcessors = new HashMap<>();
private Map<String, AdapterProcessor> canalAdapterProcessors = new HashMap<>();
private ExtensionLoader<OuterAdapter> loader;
private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){
this.canalClientConfig = canalClientConfig;
@@ -60,123 +57,145 @@ public class CanalAdapterLoader {
}
canalOuterAdapterGroups.add(canalOuterAdapters);
AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()+"|"+StringUtils.trimToEmpty(group.getGroupId()),
f-> new AdapterProcessor(canalClientConfig,canalAdapter.getInstance(),
group.getGroupId(), canalOuterAdapterGroups));
AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()
+ "|"
+ StringUtils.trimToEmpty(group.getGroupId()),
f -> new AdapterProcessor(canalClientConfig,
canalAdapter.getInstance(),
group.getGroupId(),
canalOuterAdapterGroups));
adapterProcessor.start();
logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId());
logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-"
+ group.getGroupId());
}
}
// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
//
// for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
// List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
// loadAdapter(c, canalOutConnectors);
// }
// canalOuterAdapterGroups.add(canalOutConnectors);
// }
// CanalAdapterWorker worker;
// if (StringUtils.isNotEmpty(canalServerHost)) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// canalServerHost,
// zkHosts,
// canalOuterAdapterGroups);
// } else if (zkHosts != null) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// zkHosts,
// canalOuterAdapterGroups);
// } else {
// throw new RuntimeException("No canal server connector found");
// }
// canalWorkers.put(canalAdapter.getInstance(), worker);
// worker.start();
// logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
// }
// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-kafka的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
//
// CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
// canalKafkaWorker.start();
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-rocketMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getAccessKey(),
// canalClientConfig.getSecretKey(),
// canalClientConfig.getFlatMessage(),
// canalClientConfig.isEnableMessageTrace(),
// canalClientConfig.getCustomizedTraceTopic(),
// canalClientConfig.getAccessChannel(),
// canalClientConfig.getNamespace());
// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
// rocketMQWorker.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-rabbitMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
// canalOuterAdapterGroups,
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
// rabbitMQWork.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// // CanalAdapterRabbitMQWork
// }
// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
//
// for (CanalClientConfig.Group connectorGroup :
// canalAdapter.getGroups()) {
// List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
// loadAdapter(c, canalOutConnectors);
// }
// canalOuterAdapterGroups.add(canalOutConnectors);
// }
// CanalAdapterWorker worker;
// if (StringUtils.isNotEmpty(canalServerHost)) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// canalServerHost,
// zkHosts,
// canalOuterAdapterGroups);
// } else if (zkHosts != null) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// zkHosts,
// canalOuterAdapterGroups);
// } else {
// throw new RuntimeException("No canal server connector found");
// }
// canalWorkers.put(canalAdapter.getInstance(), worker);
// worker.start();
// logger.info("Start adapter for canal instance: {} succeed",
// canalAdapter.getInstance());
// }
// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-kafka的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
//
// CanalAdapterKafkaWorker canalKafkaWorker = new
// CanalAdapterKafkaWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" +
// group.getGroupId(), canalKafkaWorker);
// canalKafkaWorker.start();
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode()))
// {
// // 初始化canal-client-rocketMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRocketMQWorker rocketMQWorker = new
// CanalAdapterRocketMQWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getAccessKey(),
// canalClientConfig.getSecretKey(),
// canalClientConfig.getFlatMessage(),
// canalClientConfig.isEnableMessageTrace(),
// canalClientConfig.getCustomizedTraceTopic(),
// canalClientConfig.getAccessChannel(),
// canalClientConfig.getNamespace());
// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" +
// group.getGroupId(), rocketMQWorker);
// rocketMQWorker.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode()))
// {
// // 初始化canal-client-rabbitMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRabbitMQWorker rabbitMQWork = new
// CanalAdapterRabbitMQWorker(canalClientConfig,
// canalOuterAdapterGroups,
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" +
// group.getGroupId(), rabbitMQWork);
// rabbitMQWork.start();
//
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// // CanalAdapterRabbitMQWork
// }
}
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {

4
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -15,14 +15,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Joiner;
/**

3
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java
View File

@@ -5,10 +5,11 @@ import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import com.alibaba.otter.canal.common.utils.CommonUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.utils.CommonUtils;
/**
* 远程配置监听器实现
*

3
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java
View File

@@ -3,9 +3,10 @@ package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.util.LinkedHashMap;
import java.util.Map;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
import org.apache.commons.lang.StringUtils;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
/**
* RDB表映射配置
*

6
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java
View File

@@ -1,6 +1,10 @@
package com.alibaba.otter.canal.client.adapter.rdb.service;
import java.sql.*;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;

3
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SingleDml.java
View File

@@ -4,9 +4,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import org.springframework.util.LinkedCaseInsensitiveMap;
import com.alibaba.otter.canal.client.adapter.support.Dml;
public class SingleDml {
private String destination;

11
client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/DBTest.java
View File

@@ -11,6 +11,7 @@ import org.junit.Ignore;
import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource;
@Ignore
public class DBTest {
@@ -39,8 +40,7 @@ public class DBTest {
Connection conn = dataSource.getConnection();
conn.setAutoCommit(false);
PreparedStatement pstmt = conn
.prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)");
PreparedStatement pstmt = conn.prepareStatement("insert into user (id,name,role_id,c_time,test1,test2) values (?,?,?,?,?,?)");
java.util.Date now = new java.util.Date();
for (int i = 1; i <= 10000; i++) {
@@ -67,14 +67,16 @@ public class DBTest {
// ResultSetMetaData rsm = rs.getMetaData();
// int cnt = rsm.getColumnCount();
// for (int i = 1; i <= cnt; i++) {
// System.out.println(rsm.getColumnName(i) + " " + rsm.getColumnType(i));
// System.out.println(rsm.getColumnName(i) + " " +
// rsm.getColumnType(i));
// }
// rs.close();
// stmt.close();
// PreparedStatement pstmt = conn
// .prepareStatement("insert into tb_user (id,name,role_id,c_time,test1,test2)
// .prepareStatement("insert into tb_user
// (id,name,role_id,c_time,test1,test2)
// values (?,?,?,?,?,?)");
// pstmt.setBigDecimal(1, new BigDecimal("5"));
// pstmt.setString(2, "test");
@@ -91,6 +93,7 @@ public class DBTest {
dataSource.close();
}
@SuppressWarnings("unused")
private String clob2Str(Clob clob) {
String content = "";
try (Reader is = clob.getCharacterStream(); BufferedReader buff = new BufferedReader(is)) {