chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

从表插入, 主表为简单字段

Browse Source
This commit is contained in:
mcy
2018-11-01 11:07:35 +08:00
parent 14e094b522
commit b2b44555ca
13 changed files with 447 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java Normal file
View File

@@ -0,0 +1,27 @@
package com.alibaba.otter.canal.client.adapter.es.test.sync;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.test.TestConstant;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import java.util.HashMap;
import java.util.Map;
public class Common {
public static ESAdapter init() {
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
outerAdapterConfig.setName("es");
outerAdapterConfig.setHosts(TestConstant.esHosts);
Map<String, String> properties = new HashMap<>();
properties.put("cluster.name", TestConstant.clusterNmae);
outerAdapterConfig.setProperties(properties);
ESAdapter esAdapter = new ESAdapter();
esAdapter.init(outerAdapterConfig);
return esAdapter;
}
}

57
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java Normal file
View File

@@ -0,0 +1,57 @@
package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
public class RoleSyncJoinOneTest {
private ESAdapter esAdapter;
@Before
public void init() {
AdapterConfigs.put("es", "mytest_user_join_one.yml");
esAdapter = Common.init();
}
/**
* 非子查询从表插入 (确保主表记录必须有数据)
*/
@Test
public void insertTest01() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("role");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("role_name", "admin");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin", response.getSource().get("_role_name"));
}
@After
public void after() {
esAdapter.destroy();
DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
}
}

57
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinSubTest.java Normal file
View File

@@ -0,0 +1,57 @@
package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
public class RoleSyncJoinSubTest {
private ESAdapter esAdapter;
@Before
public void init() {
AdapterConfigs.put("es", "mytest_user_join_sub.yml");
esAdapter = Common.init();
}
/**
* 非子查询从表插入 (确保主表记录必须有数据)
*/
@Test
public void insertTest01() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("role");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("role_name", "admin2");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin2", response.getSource().get("_role_name"));
}
@After
public void after() {
esAdapter.destroy();
DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
}
}

59
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java Normal file
View File

@@ -0,0 +1,59 @@
package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
public class UserSyncJoinOneTest {
private ESAdapter esAdapter;
@Before
public void init() {
AdapterConfigs.put("es", "mytest_user_join_one.yml");
esAdapter = Common.init();
}
/**
* 主表带函数插入, 数据库里内容必须和单测一致
*/
@Test
public void insertTest01() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric_", response.getSource().get("_name"));
}
@After
public void after() {
esAdapter.destroy();
DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
}
}

75
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java Normal file
View File

@@ -0,0 +1,75 @@
package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
public class UserSyncSingleTest {
private ESAdapter esAdapter;
@Before
public void init() {
AdapterConfigs.put("es", "mytest_user_single.yml");
esAdapter = Common.init();
}
/**
* 单表插入
*/
@Test
public void insertTest01() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric", response.getSource().get("_name"));
}
@Test
public void ttt() {
SearchResponse response = esAdapter.getTransportClient()
.prepareSearch("mytest_user")
.setTypes("_doc")
.setQuery(QueryBuilders.termQuery("_id", 1L))
.setSize(100)
.get();
for (SearchHit hit : response.getHits()) {
System.out.println(hit.getSourceAsMap().get("name"));
}
}
@After
public void after() {
esAdapter.destroy();
DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
}
}

133
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncTest.java
View File

@@ -1,133 +0,0 @@
package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
import com.alibaba.otter.canal.client.adapter.es.test.TestConstant;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
public class UserSyncTest {
private ESAdapter esAdapter;
@Before
public void init() {
AdapterConfigs.put("es", "mytest_user.yml");
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
outerAdapterConfig.setName("es");
outerAdapterConfig.setHosts(TestConstant.esHosts);
Map<String, String> properties = new HashMap<>();
properties.put("cluster.name", TestConstant.clusterNmae);
outerAdapterConfig.setProperties(properties);
esAdapter = new ESAdapter();
esAdapter.init(outerAdapterConfig);
}
/**
* 单表插入
*/
@Test
public void insertTest01() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric", response.getSource().get("name"));
}
/**
* 主表带函数插入, 数据库里内容必须和单测一致
*/
@Test
public void insertTest02() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric_", response.getSource().get("name"));
}
@Test
public void insertTest03() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("role");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("role_name", "admin");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
}
@Test
public void ttt(){
SearchResponse response = esAdapter.getTransportClient().prepareSearch("mytest_user")
.setTypes("_doc")
.setQuery(QueryBuilders.termQuery("_id", 1L))
.setSize(100)
.get();
for (SearchHit hit : response.getHits()) {
System.out.println(hit.getSourceAsMap().get("name"));
}
}
@After
public void after() {
esAdapter.destroy();
DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
}
}

10
client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_one.yml Normal file
View File

@@ -0,0 +1,10 @@
dataSourceKey: defaultDS
destination: example
esMapping:
_index: mytest_user
_type: _doc
_id: _id
sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
b.role_name as _role_name, a.c_time as _c_time from user a
left join role b on b.id=a.role_id"
commitBatch: 3000

10
client-adapter/elasticsearch/src/test/resources/es/mytest_user_join_sub.yml Normal file
View File

@@ -0,0 +1,10 @@
dataSourceKey: defaultDS
destination: example
esMapping:
_index: mytest_user
_type: _doc
_id: _id
sql: "select a.id as _id, concat(a.name,'_') as _name, a.role_id as _role_id,
b.role_name as _role_name, a.c_time as _c_time from user a
left join (select id, role_name from role ) b on b.id=a.role_id"
commitBatch: 3000

8
client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml Normal file
View File

@@ -0,0 +1,8 @@
dataSourceKey: defaultDS
destination: example
esMapping:
_index: mytest_user
_type: _doc
_id: _id
sql: "select a.id as _id, a.names as _name, a.role_id as _role_id, a.c_time as _c_time from user a"
commitBatch: 3000