chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance clickhouse support (#4957)

Browse Source
This commit is contained in:
xander-os
2023-12-04 09:34:43 +08:00
committed by GitHub
parent ca79258e4f
commit 9baeff55e4
23 changed files with 2687 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseBatchSyncServiceTest.java Normal file
View File

@@ -0,0 +1,109 @@
package com.alibaba.otter.canal.client.adapter.clickhouse;
import com.alibaba.otter.canal.client.adapter.clickhouse.sync.Common;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.util.*;
@Ignore
public class ClickHouseBatchSyncServiceTest {
private ClickHouseAdapter clickHouseAdapter;
@Before
public void init() {
clickHouseAdapter = Common.init();
}
@Test
public void insert() throws InterruptedException {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("INSERT");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric");
data.put("role_id", 1L);
data.put("c_time", new Date());
data.put("test1", "sdfasdfawe中国asfwef");
dml.setData(dataList);
clickHouseAdapter.sync(Collections.singletonList(dml));
Thread.sleep(10000L);
}
@Test
public void update() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("UPDATE");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric2");
dml.setData(dataList);
List<Map<String, Object>> oldList = new ArrayList<>();
Map<String, Object> old = new LinkedHashMap<>();
oldList.add(old);
old.put("name", "Eric");
dml.setOld(oldList);
clickHouseAdapter.sync(Collections.singletonList(dml));
}
@Test
public void delete() {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("DELETE");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric2");
dml.setData(dataList);
clickHouseAdapter.sync(Collections.singletonList(dml));
}
@Test
public void truncate() throws InterruptedException {
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType("TRUNCATE");
dml.setDatabase("mytest");
dml.setTable("user");
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", 1L);
data.put("name", "Eric2");
dml.setData(dataList);
List<Map<String, Object>> oldList = new ArrayList<>();
Map<String, Object> old = new LinkedHashMap<>();
oldList.add(old);
old.put("name", "Eric");
dml.setOld(oldList);
clickHouseAdapter.sync(Collections.singletonList(dml));
Thread.sleep(1000L);
}
}

79
client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/ClickHouseBatchSyncThreadSafeTest.java Normal file
View File

@@ -0,0 +1,79 @@
package com.alibaba.otter.canal.client.adapter.clickhouse;
/**
* @author: Xander
* @date: Created in 2023/11/13 22:27
* @email: zhrunxin33@gmail.com
* @description: Testing thread safe
*/
import ch.qos.logback.classic.Level;
import com.alibaba.otter.canal.client.adapter.clickhouse.sync.Common;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
@Ignore
public class ClickHouseBatchSyncThreadSafeTest {
private ClickHouseAdapter clickHouseAdapter;
private ExecutorService executorService;
private String[] operations = new String[]{"INSERT", "UPDATE"};
private String[] tables = new String[]{"user", "customer"};
@Before
public void init() {
clickHouseAdapter = Common.init();
Common.setLogLevel(Level.INFO);
executorService = Executors.newFixedThreadPool(5);
}
@Test
public void test01() throws InterruptedException, ExecutionException {
ArrayList<Future> list = new ArrayList();
AtomicInteger count = new AtomicInteger();
for (int i = 0; i < 10; i++) {
list.add(executorService.submit(() -> {
for (int j = 0; j < 300; j++) {
Random random = new Random();
int cou = count.incrementAndGet();
// test insert
String dmlType = operations[random.nextInt(1)];
Dml dml = new Dml();
dml.setDestination("example");
dml.setTs(new Date().getTime());
dml.setType(dmlType);
dml.setDatabase("mytest");
dml.setTable(tables[(int)Math.round(Math.random())]);
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> data = new LinkedHashMap<>();
dataList.add(data);
data.put("id", cou);
data.put("name", "Eric"+cou);
data.put("role_id", cou);
data.put("c_time", new Date());
data.put("test1", "sdfasdfawe中国asfwef");
dml.setData(dataList);
clickHouseAdapter.sync(Collections.singletonList(dml));
}
}));
}
for (Future future : list) {
future.get();
}
Thread.sleep(10000L); // waiting multiple threads execute successfully.
}
}

37
client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/TestConstant.java Normal file
View File

@@ -0,0 +1,37 @@
package com.alibaba.otter.canal.client.adapter.clickhouse;
import com.alibaba.druid.pool.DruidDataSource;
import java.sql.SQLException;
public class TestConstant {
public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
public final static String jdbcUser = "root";
public final static String jdbcPassword = "121212";
public final static DruidDataSource dataSource;
static {
dataSource = new DruidDataSource();
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(jdbcUrl);
dataSource.setUsername(jdbcUser);
dataSource.setPassword(jdbcPassword);
dataSource.setInitialSize(1);
dataSource.setMinIdle(1);
dataSource.setMaxActive(1);
dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000);
dataSource.setPoolPreparedStatements(false);
dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
dataSource.setValidationQuery("select 1");
try {
dataSource.init();
} catch (SQLException e) {
e.printStackTrace();
}
}
}

49
client-adapter/clickhouse/src/test/java/com/alibaba/otter/canal/client/adapter/clickhouse/sync/Common.java Normal file
View File

@@ -0,0 +1,49 @@
package com.alibaba.otter.canal.client.adapter.clickhouse.sync;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import com.alibaba.otter.canal.client.adapter.clickhouse.ClickHouseAdapter;
import com.alibaba.otter.canal.client.adapter.clickhouse.TestConstant;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
/**
* @author: Xander
* @date: Created in 2023/11/13 0:16
* @email: zhrunxin33@gmail.com
* @description：
*/
public class Common {
public static ClickHouseAdapter init() {
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
outerAdapterConfig.setName("clickhouse");
outerAdapterConfig.setKey("clickhouse1");
Map<String, String> properties = new HashMap<>();
properties.put("jdbc.driveClassName", "ru.yandex.clickhouse.ClickHouseDriver");
properties.put("jdbc.url", "jdbc:clickhouse://127.0.0.1:8123/default");
properties.put("jdbc.username", "default");
properties.put("jdbc.password", "123456");
outerAdapterConfig.setProperties(properties);
ClickHouseAdapter adapter = new ClickHouseAdapter();
adapter.init(outerAdapterConfig, null);
return adapter;
}
/**
* set global log level
*
* @param logLevel
*/
public static void setLogLevel(Level logLevel) {
ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
logger.setLevel(logLevel);
}
}

31
client-adapter/clickhouse/src/test/resources/clickhouse/mytest_customer.yml Normal file
View File

@@ -0,0 +1,31 @@
dataSourceKey: defaultDS
destination: example
groupId: g1
outerAdapterKey: clickhouse1
concurrent: true
dbMapping:
database: mytest
table: customer
targetTable: mytest.customer
targetPk:
id: id
# mapAll: true
targetColumns:
id:
name:
role_id:
c_time:
test1:
etlCondition: "where c_time>={}"
commitBatch: 3000 # 批量提交的大小
## Mirror schema synchronize config
#dataSourceKey: defaultDS
#destination: example
#groupId: g1
#outerAdapterKey: mysql1
#concurrent: true
#dbMapping:
# mirrorDb: true
# database: mytest

31
client-adapter/clickhouse/src/test/resources/clickhouse/mytest_user.yml Normal file
View File

@@ -0,0 +1,31 @@
dataSourceKey: defaultDS
destination: example
groupId: g1
outerAdapterKey: clickhouse1
concurrent: true
dbMapping:
database: mytest
table: user
targetTable: mytest.user
targetPk:
id: id
# mapAll: true
targetColumns:
id:
name:
role_id:
c_time:
test1:
etlCondition: "where c_time>={}"
commitBatch: 3000 # 批量提交的大小
## Mirror schema synchronize config
#dataSourceKey: defaultDS
#destination: example
#groupId: g1
#outerAdapterKey: mysql1
#concurrent: true
#dbMapping:
# mirrorDb: true
# database: mytest