chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

rdb etl count destination 接口适配

Browse Source 
es hbase 多源适配
This commit is contained in:
mcy
2018-11-12 15:07:46 +08:00
parent 7cca8857ff
commit 66912d3d05
18 changed files with 333 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java
View File

@@ -23,8 +23,7 @@ public class ConfigLoadTest {
@Test
public void testLoad() {
ESSyncConfigLoader.load();
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.getEsSyncConfig();
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load("es");
ESSyncConfig config = configMap.get("mytest_user.yml");
Assert.assertNotNull(config);
Assert.assertEquals("defaultDS", config.getDataSourceKey());
@@ -34,7 +33,8 @@ public class ConfigLoadTest {
Assert.assertEquals("id", esMapping.get_id());
Assert.assertNotNull(esMapping.getSql());
Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = ESSyncConfigLoader.getDbTableEsSyncConfig();
Assert.assertFalse(dbTableEsSyncConfig.isEmpty());
// Map<String, List<ESSyncConfig>> dbTableEsSyncConfig =
// ESSyncConfigLoader.getDbTableEsSyncConfig();
// Assert.assertFalse(dbTableEsSyncConfig.isEmpty());
}
}

37
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
@@ -32,9 +33,9 @@ public class LabelSyncJoinSub2Test {
@Test
public void test01() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"delete from label where id=1 or id=2");
Common.sqlExe(ds,"insert into label (id,user_id,label) values (1,1,'a')");
Common.sqlExe(ds,"insert into label (id,user_id,label) values (2,1,'b')");
Common.sqlExe(ds, "delete from label where id=1 or id=2");
Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
Dml dml = new Dml();
dml.setDestination("example");
@@ -46,12 +47,15 @@ public class LabelSyncJoinSub2Test {
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 2L);
data.put("user_id",1L);
data.put("user_id", 1L);
data.put("label", "b");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;a_", response.getSource().get("_labels"));
@@ -63,7 +67,7 @@ public class LabelSyncJoinSub2Test {
@Test
public void test02() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"update label set label='aa' where id=1");
Common.sqlExe(ds, "update label set label='aa' where id=1");
Dml dml = new Dml();
dml.setDestination("example");
@@ -75,7 +79,7 @@ public class LabelSyncJoinSub2Test {
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("user_id",1L);
data.put("user_id", 1L);
data.put("label", "aa");
dml.setData(dataList);
@@ -85,7 +89,11 @@ public class LabelSyncJoinSub2Test {
old.put("label", "v");
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
@@ -97,7 +105,7 @@ public class LabelSyncJoinSub2Test {
@Test
public void test03() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"delete from label where id=1");
Common.sqlExe(ds, "delete from label where id=1");
Dml dml = new Dml();
dml.setDestination("example");
@@ -109,12 +117,15 @@ public class LabelSyncJoinSub2Test {
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("user_id",1L);
data.put("user_id", 1L);
data.put("label", "a");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b_", response.getSource().get("_labels"));

37
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
@@ -32,9 +33,9 @@ public class LabelSyncJoinSubTest {
@Test
public void test01() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"delete from label where id=1 or id=2");
Common.sqlExe(ds,"insert into label (id,user_id,label) values (1,1,'a')");
Common.sqlExe(ds,"insert into label (id,user_id,label) values (2,1,'b')");
Common.sqlExe(ds, "delete from label where id=1 or id=2");
Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
Dml dml = new Dml();
dml.setDestination("example");
@@ -46,12 +47,15 @@ public class LabelSyncJoinSubTest {
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 2L);
data.put("user_id",1L);
data.put("user_id", 1L);
data.put("label", "b");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;a", response.getSource().get("_labels"));
@@ -63,7 +67,7 @@ public class LabelSyncJoinSubTest {
@Test
public void test02() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"update label set label='aa' where id=1");
Common.sqlExe(ds, "update label set label='aa' where id=1");
Dml dml = new Dml();
dml.setDestination("example");
@@ -75,7 +79,7 @@ public class LabelSyncJoinSubTest {
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("user_id",1L);
data.put("user_id", 1L);
data.put("label", "aa");
dml.setData(dataList);
@@ -85,7 +89,11 @@ public class LabelSyncJoinSubTest {
old.put("label", "a");
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b;aa", response.getSource().get("_labels"));
@@ -97,7 +105,7 @@ public class LabelSyncJoinSubTest {
@Test
public void test03() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"delete from label where id=1");
Common.sqlExe(ds, "delete from label where id=1");
Dml dml = new Dml();
dml.setDestination("example");
@@ -109,12 +117,15 @@ public class LabelSyncJoinSubTest {
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("user_id",1L);
data.put("user_id", 1L);
data.put("label", "a");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("b", response.getSource().get("_labels"));

20
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
@@ -32,8 +33,8 @@ public class RoleSyncJoinOne2Test {
@Test
public void test01() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"delete from role where id=1");
Common.sqlExe(ds,"insert into role (id,role_name) values (1,'admin')");
Common.sqlExe(ds, "delete from role where id=1");
Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
Dml dml = new Dml();
dml.setDestination("example");
@@ -46,10 +47,13 @@ public class RoleSyncJoinOne2Test {
dataList.add(data);
data.put("id", 1L);
data.put("role_name", "admin");
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin_", response.getSource().get("_role_name"));
@@ -61,7 +65,7 @@ public class RoleSyncJoinOne2Test {
@Test
public void test02() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"update role set role_name='admin3' where id=1");
Common.sqlExe(ds, "update role set role_name='admin3' where id=1");
Dml dml = new Dml();
dml.setDestination("example");
@@ -82,7 +86,11 @@ public class RoleSyncJoinOne2Test {
old.put("role_name", "admin");
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin3_", response.getSource().get("_role_name"));

68
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java
View File

@@ -4,6 +4,7 @@ import java.util.*;
import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import org.elasticsearch.action.get.GetResponse;
import org.junit.Assert;
import org.junit.Before;
@@ -47,7 +48,11 @@ public class RoleSyncJoinOneTest {
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin", response.getSource().get("_role_name"));
@@ -80,7 +85,11 @@ public class RoleSyncJoinOneTest {
old.put("role_name", "admin");
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin2", response.getSource().get("_role_name"));
@@ -96,28 +105,32 @@ public class RoleSyncJoinOneTest {
Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')");
Common.sqlExe(ds, "update user set role_id=2 where id=1");
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("UPDATE");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("role_id", 2L);
dml.setData(dataList);
List<Map<String, Object>> oldList = new ArrayList<>();
Map<String, Object> old = new LinkedHashMap<>();
oldList.add(old);
old.put("role_id", 1L);
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("UPDATE");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("role_id", 2L);
dml.setData(dataList);
List<Map<String, Object>> oldList = new ArrayList<>();
Map<String, Object> old = new LinkedHashMap<>();
oldList.add(old);
old.put("role_id", 1L);
dml.setOld(oldList);
GetResponse response =
esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("operator", response.getSource().get("_role_name"));
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("operator", response.getSource().get("_role_name"));
Common.sqlExe(ds, "update user set role_id=1 where id=1");
@@ -138,7 +151,8 @@ public class RoleSyncJoinOneTest {
oldList2.add(old2);
old2.put("role_id", 2L);
dml2.setOld(oldList2);
esAdapter.getEsSyncService().sync(dml2);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml2);
GetResponse response2 = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
@@ -166,7 +180,11 @@ public class RoleSyncJoinOneTest {
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertNull(response.getSource().get("_role_name"));

20
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java
View File

@@ -4,6 +4,7 @@ import java.util.*;
import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
@@ -32,8 +33,8 @@ public class UserSyncJoinOneTest {
@Test
public void test01() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"delete from user where id=1");
Common.sqlExe(ds,"insert into user (id,name,role_id) values (1,'Eric',1)");
Common.sqlExe(ds, "delete from user where id=1");
Common.sqlExe(ds, "insert into user (id,name,role_id) values (1,'Eric',1)");
Dml dml = new Dml();
dml.setDestination("example");
@@ -48,10 +49,13 @@ public class UserSyncJoinOneTest {
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric_", response.getSource().get("_name"));
@@ -63,7 +67,7 @@ public class UserSyncJoinOneTest {
@Test
public void test02() {
DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
Common.sqlExe(ds,"update user set name='Eric2' where id=1");
Common.sqlExe(ds, "update user set name='Eric2' where id=1");
Dml dml = new Dml();
dml.setDestination("example");
@@ -83,7 +87,11 @@ public class UserSyncJoinOneTest {
old.put("name", "Eric");
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric2_", response.getSource().get("_name"));

21
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java
View File

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.es.test.sync;
import java.util.*;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import org.elasticsearch.action.get.GetResponse;
import org.junit.After;
import org.junit.Assert;
@@ -42,10 +43,13 @@ public class UserSyncSingleTest {
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric", response.getSource().get("_name"));
@@ -74,7 +78,11 @@ public class UserSyncSingleTest {
old.put("name", "Eric");
dml.setOld(oldList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertEquals("Eric2", response.getSource().get("_name"));
@@ -98,10 +106,13 @@ public class UserSyncSingleTest {
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
dml.setData(dataList);
esAdapter.getEsSyncService().sync(dml);
String database = dml.getDatabase();
String table = dml.getTable();
List<ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
esAdapter.getEsSyncService().sync(esSyncConfigs, dml);
GetResponse response = esAdapter.getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
Assert.assertNull(response.getSource());