chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

modify

Browse Source
This commit is contained in:
mcy
2018-10-31 18:47:16 +08:00
parent 94a3cfa9de
commit 14e094b522
7 changed files with 110 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java
View File

@@ -23,7 +23,7 @@ public class SchemaItem {
this.isAllFieldsSimple(); this.isAllFieldsSimple();
aliasTableItems.values().forEach(tableItem -> { aliasTableItems.values().forEach(tableItem -> {
tableItem.getRelationTableFields(); tableItem.getRelationTableFields();
tableItem.getRelationKeyFieldItems(); // tableItem.getRelationKeyFieldItems();
tableItem.getRelationSelectFieldItems(); tableItem.getRelationSelectFieldItems();
}); });
} }
@@ -145,8 +145,9 @@ public class SchemaItem {
private boolean main; private boolean main;
private boolean subQuery; private boolean subQuery;
private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段 private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段对应主表查询字段
private volatile List<FieldItem> relationKeyFieldItems; // 关联条件字段在select中的对应字段 // private volatile List<FieldItem> relationKeyFieldItems; //
// 关联条件字段在select中的对应字段
private volatile List<FieldItem> relationSelectFieldItems; // 子表所在主表的查询字段 private volatile List<FieldItem> relationSelectFieldItems; // 子表所在主表的查询字段
public TableItem(SchemaItem schemaItem){ public TableItem(SchemaItem schemaItem){
@@ -267,34 +268,36 @@ public class SchemaItem {
return relationTableFields; return relationTableFields;
} }
public List<FieldItem> getRelationKeyFieldItems() { // public List<FieldItem> getRelationKeyFieldItems() {
if (relationKeyFieldItems == null) { // if (relationKeyFieldItems == null) {
synchronized (SchemaItem.class) { // synchronized (SchemaItem.class) {
if (relationKeyFieldItems == null) { // if (relationKeyFieldItems == null) {
relationKeyFieldItems = new ArrayList<>(); // relationKeyFieldItems = new ArrayList<>();
getRelationFields().forEach(relationFieldsPair -> { // getRelationFields().forEach(relationFieldsPair -> {
FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); // FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields() // List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
.get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); // .get(leftFieldItem.getOwner() + "." +
if (selectFieldItem != null && !selectFieldItem.isEmpty()) { // leftFieldItem.getColumn().getColumnName());
relationKeyFieldItems.addAll(selectFieldItem); // if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
} else { // relationKeyFieldItems.addAll(selectFieldItem);
FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); // } else {
selectFieldItem = getSchemaItem().getColumnFields() // FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
.get(rightFieldItem.getOwner() + "." + rightFieldItem.getColumn().getColumnName()); // selectFieldItem = getSchemaItem().getColumnFields()
if (selectFieldItem != null && !selectFieldItem.isEmpty()) { // .get(rightFieldItem.getOwner() + "." +
relationKeyFieldItems.addAll(selectFieldItem); // rightFieldItem.getColumn().getColumnName());
} else { // if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
throw new UnsupportedOperationException( // relationKeyFieldItems.addAll(selectFieldItem);
"Relation condition column must in select columns."); // } else {
} // throw new UnsupportedOperationException(
} // "Relation condition column must in select columns.");
}); // }
} // }
} // });
} // }
return relationKeyFieldItems; // }
} // }
// return relationKeyFieldItems;
// }
public List<FieldItem> getRelationSelectFieldItems() { public List<FieldItem> getRelationSelectFieldItems() {
if (relationSelectFieldItems == null) { if (relationSelectFieldItems == null) {

39
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java
View File

@@ -190,41 +190,46 @@ public class ESSyncService {
if (allFieldsSimple) { if (allFieldsSimple) {
// 不是子查询 // 不是子查询
if (!tableItem.isSubQuery()) { if (!tableItem.isSubQuery()) {
Map<String, Object> esFieldData = new LinkedHashMap<>(); Map<String, Object> esFieldData = new LinkedHashMap<>();
for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) { for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
Object value = esTemplate Object value = esTemplate
.getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); .getValFromData(mapping, data, fieldItem.getColumn().getColumnName());
esFieldData.put(fieldItem.getFieldName(), value); esFieldData.put(fieldItem.getFieldName(), value);
} }
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields() for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields()
.entrySet()) { .entrySet()) {
Object value = esTemplate Object value = esTemplate
.getValFromData(mapping, data, entry.getKey().getColumn().getColumnName()); .getValFromData(mapping, data, entry.getKey().getColumn().getColumnName());
for (FieldItem fieldItem : entry.getValue()) { for (FieldItem fieldItem : entry.getValue()) {
queryBuilder.must(QueryBuilders.termsQuery(fieldItem.getFieldName(), value)); String fieldName = fieldItem.getFieldName();
// 判断是否是主键
if (fieldName.equals(mapping.get_id())) {
fieldName = "_id";
}
queryBuilder.must(QueryBuilders.termsQuery(fieldName, value));
} }
} }
// if (logger.isDebugEnabled()) {
// for (FieldItem fieldItem : tableItem.getRelationTableFields()) { logger.trace(
// Object value = esTemplate "Join table update es index by foreign key, destination:{}, table: {}, index: {}",
// .getValFromData(mapping, data, fieldItem.getColumn().getColumnName()); config.getDestination(),
// dml.getTable(),
// } mapping.get_index());
}
// if (logger.isDebugEnabled()) {
// logger.debug("从表insert, 且均为简单字段，对es进行update_by_query, queryBuilder: {},
// esFieldData:{}", queryBuilder.toString(), esFieldData);
// }
boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData); boolean result = esTemplate.updateByQuery(mapping, queryBuilder, esFieldData);
// if (!result) { if (!result) {
// logger.error("从表insert，均为简单字段，直接对es进行update_by_query, es更新存在错误, table: {}, logger.error(
// index: {}, dml: {}", dml.getTable(), config.getEsSyn().getIndex(), dml); "Join table update es index by foreign key error, destination:{}, table: {}, index: {}",
// } config.getDestination(),
dml.getTable(),
mapping.get_index());
} }
} else {
// TODO // TODO
}
} else { } else {
// TODO 查询总sql // TODO 查询总sql
} }

4
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java
View File

@@ -264,7 +264,7 @@ public class ESTemplate {
if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
logger.warn(itemResponse.getFailureMessage()); logger.warn(itemResponse.getFailureMessage());
} else { } else {
logger.error("ES 同步数据错误 {}", itemResponse.getFailureMessage()); logger.error("ES sync commit error: {}", itemResponse.getFailureMessage());
} }
} }
} }
@@ -314,7 +314,7 @@ public class ESTemplate {
value = resultSet.getByte(fieldName); value = resultSet.getByte(fieldName);
} }
} }
return value; return ESSyncUtil.typeConvert(value, esType);
} }
public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,

7
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java
View File

@@ -39,8 +39,9 @@ public class SqlParseTest {
.forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName())); .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName()));
// 获取关联字段在select中的对应字段 // 获取关联字段在select中的对应字段
List<FieldItem> relationSelectFieldItem = tableItem.getRelationKeyFieldItems(); // List<FieldItem> relationSelectFieldItem =
relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels", // tableItem.getRelationKeyFieldItems();
fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName())); // relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels",
// fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName()));
} }
} }

17
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java
View File

@@ -3,6 +3,9 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*; import java.util.*;
import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@@ -53,6 +56,7 @@ public class UserSyncTest {
dataList.add(data); dataList.add(data);
data.put("id", 1L); data.put("id", 1L);
data.put("name", "Eric"); data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date()); data.put("c_time", new Date());
dml.setData(dataList); dml.setData(dataList);
@@ -79,6 +83,7 @@ public class UserSyncTest {
dataList.add(data); dataList.add(data);
data.put("id", 1L); data.put("id", 1L);
data.put("name", "Eric"); data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date()); data.put("c_time", new Date());
dml.setData(dataList); dml.setData(dataList);
@@ -108,6 +113,18 @@ public class UserSyncTest {
esAdapter.getEsSyncService().sync(dml); esAdapter.getEsSyncService().sync(dml);
} }
@Test
public void ttt(){
SearchResponse response = esAdapter.getTransportClient().prepareSearch("mytest_user")
.setTypes("_doc")
.setQuery(QueryBuilders.termQuery("_id", 1L))
.setSize(100)
.get();
for (SearchHit hit : response.getHits()) {
System.out.println(hit.getSourceAsMap().get("name"));
}
}
@After @After
public void after() { public void after() {
esAdapter.destroy(); esAdapter.destroy();

13
client-adapter/elasticsearch/src/test/resources/log4j2-test.xml Normal file
View File

@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="ERROR">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>

34
client-adapter/launcher/src/main/resources/application.yml
View File

@@ -16,34 +16,34 @@ hbase.zookeeper.property.clientPort: 2181
hbase.zookeeper.znode.parent: /hbase hbase.zookeeper.znode.parent: /hbase
canal.conf: canal.conf:
# canalServerHost: 127.0.0.1:11111 canalServerHost: 127.0.0.1:11111
# zookeeperHosts: slave1:2181 # zookeeperHosts: slave1:2181
bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876 # bootstrapServers: slave1:6667 #or rocketmq nameservers:host1:9876;host2:9876
flatMessage: true flatMessage: true
retry: 3 retry: 3
timeout: 20000 timeout: 20000
batchSize: 50 batchSize: 50
# canalInstances: canalInstances:
# - instance: example - instance: example
# groups: groups:
# - outAdapters: - outAdapters:
# - name: es - name: es
# hosts: 127.0.0.1:9300 hosts: 127.0.0.1:9300
# properties: properties:
# cluster.name: elasticsearch cluster.name: elasticsearch
# - name: logger # - name: logger
# - name: hbase # - name: hbase
# properties: # properties:
# hbase.zookeeper.quorum: ${hbase.zookeeper.quorum} # hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
# hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort} # hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
# zookeeper.znode.parent: ${hbase.zookeeper.znode.parent} # zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
mqTopics: # mqTopics:
- mqMode: kafka # - mqMode: kafka
topic: example # topic: example
groups: # groups:
- groupId: g2 # - groupId: g2
outAdapters: # outAdapters:
- name: logger # - name: logger
# mqTopics: # mqTopics:
# - mqMode: rocketmq # - mqMode: rocketmq
# topic: example # topic: example