mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed format
This commit is contained in:
@@ -1,5 +1,16 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu;
|
package com.alibaba.otter.canal.client.adapter.kudu;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
||||||
@@ -11,12 +22,6 @@ import com.alibaba.otter.canal.client.adapter.support.Dml;
|
|||||||
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
@@ -24,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
*/
|
*/
|
||||||
@SPI("kudu")
|
@SPI("kudu")
|
||||||
public class KuduAdapter implements OuterAdapter {
|
public class KuduAdapter implements OuterAdapter {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(KuduAdapter.class);
|
private static Logger logger = LoggerFactory.getLogger(KuduAdapter.class);
|
||||||
|
|
||||||
private Map<String, KuduMappingConfig> kuduMapping = new ConcurrentHashMap<>(); // 文件名对应配置
|
private Map<String, KuduMappingConfig> kuduMapping = new ConcurrentHashMap<>(); // 文件名对应配置
|
||||||
@@ -47,7 +53,6 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
return mappingConfigCache;
|
return mappingConfigCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void init(OuterAdapterConfig configuration, Properties envProperties) {
|
public void init(OuterAdapterConfig configuration, Properties envProperties) {
|
||||||
this.envProperties = envProperties;
|
this.envProperties = envProperties;
|
||||||
@@ -61,7 +66,7 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
dataSourceKey = mappingConfig.getDataSourceKey();
|
dataSourceKey = mappingConfig.getDataSourceKey();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
//判断目标字段是否为空
|
// 判断目标字段是否为空
|
||||||
if (kuduMapping.isEmpty()) {
|
if (kuduMapping.isEmpty()) {
|
||||||
throw new RuntimeException("No kudu adapter found for config key: " + configuration.getKey());
|
throw new RuntimeException("No kudu adapter found for config key: " + configuration.getKey());
|
||||||
}
|
}
|
||||||
@@ -72,12 +77,10 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
||||||
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
|
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
|
||||||
+ StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
|
+ StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
|
||||||
+ mappingConfig.getKuduMapping().getDatabase() + "-"
|
+ mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
|
||||||
+ mappingConfig.getKuduMapping().getTable();
|
|
||||||
} else {
|
} else {
|
||||||
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
|
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
|
||||||
+ mappingConfig.getKuduMapping().getDatabase() + "-"
|
+ mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
|
||||||
+ mappingConfig.getKuduMapping().getTable();
|
|
||||||
}
|
}
|
||||||
Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
|
Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
|
||||||
k1 -> new ConcurrentHashMap<>());
|
k1 -> new ConcurrentHashMap<>());
|
||||||
@@ -108,9 +111,7 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
String table = dml.getTable();
|
String table = dml.getTable();
|
||||||
Map<String, KuduMappingConfig> configMap;
|
Map<String, KuduMappingConfig> configMap;
|
||||||
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
||||||
configMap = mappingConfigCache.get(destination + "-"
|
configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table);
|
||||||
+ groupId + "_"
|
|
||||||
+ database + "-" + table);
|
|
||||||
} else {
|
} else {
|
||||||
configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
|
configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
|
||||||
}
|
}
|
||||||
@@ -131,7 +132,8 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
logger.error("groupID didn't mach,please check your gruopId ");
|
logger.error("groupID didn't mach,please check your gruopId ");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-" + table);
|
logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-"
|
||||||
|
+ table);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -141,7 +143,7 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
if (kuduConfigMonitor != null) {
|
if (kuduConfigMonitor != null) {
|
||||||
kuduConfigMonitor.destroy();
|
kuduConfigMonitor.destroy();
|
||||||
}
|
}
|
||||||
//加入kudu client 关闭钩子
|
// 加入kudu client 关闭钩子
|
||||||
kuduTemplate.closeKuduClient();
|
kuduTemplate.closeKuduClient();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,16 +1,18 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description kudu配置文件映射
|
* @description kudu配置文件映射
|
||||||
*/
|
*/
|
||||||
public class KuduMappingConfig implements AdapterConfig {
|
public class KuduMappingConfig implements AdapterConfig {
|
||||||
|
|
||||||
private String dataSourceKey; // 数据源key
|
private String dataSourceKey; // 数据源key
|
||||||
|
|
||||||
private String destination; // canal实例或MQ的topic
|
private String destination; // canal实例或MQ的topic
|
||||||
@@ -77,7 +79,6 @@ public class KuduMappingConfig implements AdapterConfig {
|
|||||||
this.destination = destination;
|
this.destination = destination;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void validate() {
|
public void validate() {
|
||||||
if (kuduMapping.database == null || kuduMapping.database.isEmpty()) {
|
if (kuduMapping.database == null || kuduMapping.database.isEmpty()) {
|
||||||
throw new NullPointerException("KuduMapping.database");
|
throw new NullPointerException("KuduMapping.database");
|
||||||
|
|||||||
@@ -1,19 +1,21 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
|
||||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description kudu表信息加载
|
* @description kudu表信息加载
|
||||||
*/
|
*/
|
||||||
public class KuduMappingConfigLoader {
|
public class KuduMappingConfigLoader {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(KuduMappingConfigLoader.class);
|
private static Logger logger = LoggerFactory.getLogger(KuduMappingConfigLoader.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -28,8 +30,11 @@ public class KuduMappingConfigLoader {
|
|||||||
|
|
||||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
|
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
|
||||||
configContentMap.forEach((fileName, content) -> {
|
configContentMap.forEach((fileName, content) -> {
|
||||||
KuduMappingConfig config = YmlConfigBinder
|
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||||
.bindYmlToObj(null, content, KuduMappingConfig.class, null, envProperties);
|
content,
|
||||||
|
KuduMappingConfig.class,
|
||||||
|
null,
|
||||||
|
envProperties);
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.monitor;
|
package com.alibaba.otter.canal.client.adapter.kudu.monitor;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
import java.io.File;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
import java.util.HashMap;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import java.util.Map;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
import java.util.Properties;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
|
||||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||||
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
|
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
|
||||||
import org.apache.commons.io.monitor.FileAlterationMonitor;
|
import org.apache.commons.io.monitor.FileAlterationMonitor;
|
||||||
@@ -13,16 +13,18 @@ import org.apache.commons.lang.StringUtils;
|
|||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.File;
|
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||||
import java.util.HashMap;
|
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||||
import java.util.Map;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import java.util.Properties;
|
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description 配置文件监听
|
* @description 配置文件监听
|
||||||
*/
|
*/
|
||||||
public class KuduConfigMonitor {
|
public class KuduConfigMonitor {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(KuduConfigMonitor.class);
|
private static final Logger logger = LoggerFactory.getLogger(KuduConfigMonitor.class);
|
||||||
|
|
||||||
private static final String adapterName = "kudu";
|
private static final String adapterName = "kudu";
|
||||||
@@ -64,6 +66,7 @@ public class KuduConfigMonitor {
|
|||||||
* 配置文件监听
|
* 配置文件监听
|
||||||
*/
|
*/
|
||||||
private class FileListener extends FileAlterationListenerAdaptor {
|
private class FileListener extends FileAlterationListenerAdaptor {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onFileCreate(File file) {
|
public void onFileCreate(File file) {
|
||||||
super.onFileCreate(file);
|
super.onFileCreate(file);
|
||||||
@@ -71,8 +74,11 @@ public class KuduConfigMonitor {
|
|||||||
try {
|
try {
|
||||||
// 加载新增的配置文件
|
// 加载新增的配置文件
|
||||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||||
KuduMappingConfig config = YmlConfigBinder
|
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||||
.bindYmlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
|
configContent,
|
||||||
|
KuduMappingConfig.class,
|
||||||
|
null,
|
||||||
|
envProperties);
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -92,14 +98,17 @@ public class KuduConfigMonitor {
|
|||||||
try {
|
try {
|
||||||
if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
|
if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
|
||||||
// 加载配置文件
|
// 加载配置文件
|
||||||
String configContent = MappingConfigsLoader
|
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
|
||||||
.loadConfig(adapterName + File.separator + file.getName());
|
+ file.getName());
|
||||||
if (configContent == null) {
|
if (configContent == null) {
|
||||||
onFileDelete(file);
|
onFileDelete(file);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
KuduMappingConfig config = YmlConfigBinder
|
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||||
.bindYmlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
|
configContent,
|
||||||
|
KuduMappingConfig.class,
|
||||||
|
null,
|
||||||
|
envProperties);
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -114,7 +123,6 @@ public class KuduConfigMonitor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onFileDelete(File file) {
|
public void onFileDelete(File file) {
|
||||||
super.onFileDelete(file);
|
super.onFileDelete(file);
|
||||||
|
|||||||
@@ -1,5 +1,18 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
||||||
|
|
||||||
|
import java.sql.ResultSetMetaData;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
|
||||||
|
import org.apache.kudu.client.KuduException;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.support.SyncUtil;
|
import com.alibaba.otter.canal.client.adapter.kudu.support.SyncUtil;
|
||||||
@@ -8,13 +21,6 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
|||||||
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
import org.apache.kudu.client.KuduException;
|
|
||||||
|
|
||||||
import javax.sql.DataSource;
|
|
||||||
import java.sql.ResultSetMetaData;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
@@ -25,7 +31,7 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
private KuduTemplate kuduTemplate;
|
private KuduTemplate kuduTemplate;
|
||||||
private KuduMappingConfig config;
|
private KuduMappingConfig config;
|
||||||
|
|
||||||
public KuduEtlService(KuduTemplate kuduTemplate, KuduMappingConfig config) {
|
public KuduEtlService(KuduTemplate kuduTemplate, KuduMappingConfig config){
|
||||||
super("kudu", config);
|
super("kudu", config);
|
||||||
this.kuduTemplate = kuduTemplate;
|
this.kuduTemplate = kuduTemplate;
|
||||||
this.config = config;
|
this.config = config;
|
||||||
@@ -37,7 +43,7 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
|
|
||||||
KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
|
KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
|
||||||
boolean flag = kuduTemplate.tableExists(kuduMapping.getTargetTable());
|
boolean flag = kuduTemplate.tableExists(kuduMapping.getTargetTable());
|
||||||
//表不存在,停止导入
|
// 表不存在,停止导入
|
||||||
if (!flag) {
|
if (!flag) {
|
||||||
logger.info("{} is don't hava,please check your kudu table !", kuduMapping.getTargetTable());
|
logger.info("{} is don't hava,please check your kudu table !", kuduMapping.getTargetTable());
|
||||||
errMsg.add(kuduMapping.getTargetTable() + " is don't hava,please check your kudu table !");
|
errMsg.add(kuduMapping.getTargetTable() + " is don't hava,please check your kudu table !");
|
||||||
@@ -49,12 +55,12 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
return importData(sql, params);
|
return importData(sql, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values, AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
|
protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
|
||||||
|
AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
|
||||||
KuduMappingConfig.KuduMapping kuduMapping = (KuduMappingConfig.KuduMapping) mapping;
|
KuduMappingConfig.KuduMapping kuduMapping = (KuduMappingConfig.KuduMapping) mapping;
|
||||||
//获取字段元数据
|
// 获取字段元数据
|
||||||
Map<String, String> columnsMap = new LinkedHashMap<>();//需要同步的字段
|
Map<String, String> columnsMap = new LinkedHashMap<>();// 需要同步的字段
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Util.sqlRS(ds, "SELECT * FROM " + SyncUtil.getDbTableName(kuduMapping) + " LIMIT 1", rs -> {
|
Util.sqlRS(ds, "SELECT * FROM " + SyncUtil.getDbTableName(kuduMapping) + " LIMIT 1", rs -> {
|
||||||
@@ -72,7 +78,7 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
//写入数据
|
// 写入数据
|
||||||
logger.info("etl select data sql is :{}", sql);
|
logger.info("etl select data sql is :{}", sql);
|
||||||
Util.sqlRS(ds, sql, values, rs -> {
|
Util.sqlRS(ds, sql, values, rs -> {
|
||||||
int idx = 1;
|
int idx = 1;
|
||||||
@@ -81,8 +87,8 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
while (rs.next()) {
|
while (rs.next()) {
|
||||||
Map<String, Object> data = new HashMap<>();
|
Map<String, Object> data = new HashMap<>();
|
||||||
for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
|
for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
|
||||||
String mysqlColumnName = entry.getKey();//mysql字段名
|
String mysqlColumnName = entry.getKey();// mysql字段名
|
||||||
String kuduColumnName = entry.getValue();//kudu字段名
|
String kuduColumnName = entry.getValue();// kudu字段名
|
||||||
if (kuduColumnName == null) {
|
if (kuduColumnName == null) {
|
||||||
kuduColumnName = mysqlColumnName;
|
kuduColumnName = mysqlColumnName;
|
||||||
}
|
}
|
||||||
@@ -120,7 +126,7 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
|
errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
});
|
} );
|
||||||
return true;
|
return true;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error(e.getMessage(), e);
|
logger.error(e.getMessage(), e);
|
||||||
|
|||||||
@@ -1,36 +1,42 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.kudu.client.KuduException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||||
import org.apache.kudu.client.KuduException;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description kudu实时同步
|
* @description kudu实时同步
|
||||||
*/
|
*/
|
||||||
public class KuduSyncService {
|
public class KuduSyncService {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(KuduSyncService.class);
|
private static Logger logger = LoggerFactory.getLogger(KuduSyncService.class);
|
||||||
|
|
||||||
private KuduTemplate kuduTemplate;
|
private KuduTemplate kuduTemplate;
|
||||||
|
|
||||||
// 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
|
// 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
|
||||||
// private Map<String, Map<String, Integer>> columnsTypeCache = new ConcurrentHashMap<>();
|
// private Map<String, Map<String, Integer>> columnsTypeCache = new
|
||||||
|
// ConcurrentHashMap<>();
|
||||||
|
|
||||||
public KuduSyncService(KuduTemplate kuduTemplate) {
|
public KuduSyncService(KuduTemplate kuduTemplate){
|
||||||
this.kuduTemplate = kuduTemplate;
|
this.kuduTemplate = kuduTemplate;
|
||||||
}
|
}
|
||||||
|
|
||||||
// public Map<String, Map<String, Integer>> getColumnsTypeCache() {
|
// public Map<String, Map<String, Integer>> getColumnsTypeCache() {
|
||||||
// return columnsTypeCache;
|
// return columnsTypeCache;
|
||||||
// }
|
// }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 同步事件处理
|
* 同步事件处理
|
||||||
@@ -71,7 +77,7 @@ public class KuduSyncService {
|
|||||||
if (data == null || data.isEmpty()) {
|
if (data == null || data.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
//判定主键映射
|
// 判定主键映射
|
||||||
String pkId = "";
|
String pkId = "";
|
||||||
Map<String, String> targetPk = kuduMapping.getTargetPk();
|
Map<String, String> targetPk = kuduMapping.getTargetPk();
|
||||||
for (Map.Entry<String, String> entry : targetPk.entrySet()) {
|
for (Map.Entry<String, String> entry : targetPk.entrySet()) {
|
||||||
@@ -83,7 +89,7 @@ public class KuduSyncService {
|
|||||||
pkId = kuduID;
|
pkId = kuduID;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//切割联合主键
|
// 切割联合主键
|
||||||
List<String> pkIds = Arrays.asList(pkId.split(","));
|
List<String> pkIds = Arrays.asList(pkId.split(","));
|
||||||
try {
|
try {
|
||||||
int idx = 1;
|
int idx = 1;
|
||||||
@@ -157,7 +163,6 @@ public class KuduSyncService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,16 +1,30 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
||||||
|
|
||||||
import org.apache.kudu.ColumnSchema;
|
|
||||||
import org.apache.kudu.Schema;
|
|
||||||
import org.apache.kudu.Type;
|
|
||||||
import org.apache.kudu.client.*;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.text.ParseException;
|
import java.text.ParseException;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.*;
|
import java.util.Arrays;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.kudu.ColumnSchema;
|
||||||
|
import org.apache.kudu.Schema;
|
||||||
|
import org.apache.kudu.Type;
|
||||||
|
import org.apache.kudu.client.Delete;
|
||||||
|
import org.apache.kudu.client.Insert;
|
||||||
|
import org.apache.kudu.client.KuduClient;
|
||||||
|
import org.apache.kudu.client.KuduException;
|
||||||
|
import org.apache.kudu.client.KuduScanner;
|
||||||
|
import org.apache.kudu.client.KuduSession;
|
||||||
|
import org.apache.kudu.client.KuduTable;
|
||||||
|
import org.apache.kudu.client.OperationResponse;
|
||||||
|
import org.apache.kudu.client.PartialRow;
|
||||||
|
import org.apache.kudu.client.SessionConfiguration;
|
||||||
|
import org.apache.kudu.client.Upsert;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
@@ -27,7 +41,7 @@ public class KuduTemplate {
|
|||||||
|
|
||||||
private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
|
|
||||||
public KuduTemplate(String master_str) {
|
public KuduTemplate(String master_str){
|
||||||
this.masters = master_str;
|
this.masters = master_str;
|
||||||
checkClient();
|
checkClient();
|
||||||
}
|
}
|
||||||
@@ -37,12 +51,12 @@ public class KuduTemplate {
|
|||||||
*/
|
*/
|
||||||
private void checkClient() {
|
private void checkClient() {
|
||||||
if (kuduClient == null) {
|
if (kuduClient == null) {
|
||||||
//kudu master 以逗号分隔
|
// kudu master 以逗号分隔
|
||||||
List<String> masterList = Arrays.asList(masters.split(","));
|
List<String> masterList = Arrays.asList(masters.split(","));
|
||||||
kuduClient = new KuduClient.KuduClientBuilder(masterList)
|
kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
|
||||||
.defaultOperationTimeoutMs(60000)
|
|
||||||
.defaultSocketReadTimeoutMs(60000)
|
.defaultSocketReadTimeoutMs(60000)
|
||||||
.defaultAdminOperationTimeoutMs(60000).build();
|
.defaultAdminOperationTimeoutMs(60000)
|
||||||
|
.build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,7 +90,7 @@ public class KuduTemplate {
|
|||||||
try {
|
try {
|
||||||
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
|
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
|
||||||
session.setMutationBufferSpace(OPERATION_BATCH);
|
session.setMutationBufferSpace(OPERATION_BATCH);
|
||||||
//获取元数据结构
|
// 获取元数据结构
|
||||||
Map<String, Type> metaMap = new HashMap<>();
|
Map<String, Type> metaMap = new HashMap<>();
|
||||||
Schema schema = kuduTable.getSchema();
|
Schema schema = kuduTable.getSchema();
|
||||||
for (ColumnSchema columnSchema : schema.getColumns()) {
|
for (ColumnSchema columnSchema : schema.getColumns()) {
|
||||||
@@ -92,7 +106,7 @@ public class KuduTemplate {
|
|||||||
String name = entry.getKey().toLowerCase();
|
String name = entry.getKey().toLowerCase();
|
||||||
Type type = metaMap.get(name);
|
Type type = metaMap.get(name);
|
||||||
Object value = entry.getValue();
|
Object value = entry.getValue();
|
||||||
fillRow(row, name, value, type); //填充行数据
|
fillRow(row, name, value, type); // 填充行数据
|
||||||
}
|
}
|
||||||
session.apply(delete);
|
session.apply(delete);
|
||||||
// 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
|
// 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
|
||||||
@@ -142,7 +156,7 @@ public class KuduTemplate {
|
|||||||
try {
|
try {
|
||||||
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
|
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
|
||||||
session.setMutationBufferSpace(OPERATION_BATCH);
|
session.setMutationBufferSpace(OPERATION_BATCH);
|
||||||
//获取元数据结构
|
// 获取元数据结构
|
||||||
Map<String, Type> metaMap = new HashMap<>();
|
Map<String, Type> metaMap = new HashMap<>();
|
||||||
Schema schema = kuduTable.getSchema();
|
Schema schema = kuduTable.getSchema();
|
||||||
for (ColumnSchema columnSchema : schema.getColumns()) {
|
for (ColumnSchema columnSchema : schema.getColumns()) {
|
||||||
@@ -158,7 +172,7 @@ public class KuduTemplate {
|
|||||||
String name = entry.getKey().toLowerCase();
|
String name = entry.getKey().toLowerCase();
|
||||||
Type type = metaMap.get(name);
|
Type type = metaMap.get(name);
|
||||||
Object value = entry.getValue();
|
Object value = entry.getValue();
|
||||||
fillRow(row, name, value, type); //填充行数据
|
fillRow(row, name, value, type); // 填充行数据
|
||||||
}
|
}
|
||||||
session.apply(upsert);
|
session.apply(upsert);
|
||||||
// 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
|
// 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
|
||||||
@@ -192,7 +206,6 @@ public class KuduTemplate {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -207,9 +220,10 @@ public class KuduTemplate {
|
|||||||
KuduTable kuduTable = kuduClient.openTable(tableName);// 打开表
|
KuduTable kuduTable = kuduClient.openTable(tableName);// 打开表
|
||||||
KuduSession session = kuduClient.newSession(); // 创建写session,kudu必须通过session写入
|
KuduSession session = kuduClient.newSession(); // 创建写session,kudu必须通过session写入
|
||||||
try {
|
try {
|
||||||
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // 采取Flush方式 手动刷新
|
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // 采取Flush方式
|
||||||
|
// 手动刷新
|
||||||
session.setMutationBufferSpace(OPERATION_BATCH);
|
session.setMutationBufferSpace(OPERATION_BATCH);
|
||||||
//获取元数据结构
|
// 获取元数据结构
|
||||||
Map<String, Type> metaMap = new HashMap<>();
|
Map<String, Type> metaMap = new HashMap<>();
|
||||||
Schema schema = kuduTable.getSchema();
|
Schema schema = kuduTable.getSchema();
|
||||||
for (ColumnSchema columnSchema : schema.getColumns()) {
|
for (ColumnSchema columnSchema : schema.getColumns()) {
|
||||||
@@ -225,7 +239,7 @@ public class KuduTemplate {
|
|||||||
String name = entry.getKey().toLowerCase();
|
String name = entry.getKey().toLowerCase();
|
||||||
Type type = metaMap.get(name);
|
Type type = metaMap.get(name);
|
||||||
Object value = entry.getValue();
|
Object value = entry.getValue();
|
||||||
fillRow(row, name, value, type); //填充行数据
|
fillRow(row, name, value, type); // 填充行数据
|
||||||
}
|
}
|
||||||
session.apply(insert);
|
session.apply(insert);
|
||||||
// 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
|
// 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
|
||||||
@@ -272,9 +286,9 @@ public class KuduTemplate {
|
|||||||
long rowCount = 0L;
|
long rowCount = 0L;
|
||||||
try {
|
try {
|
||||||
KuduTable kuduTable = kuduClient.openTable(tableName);
|
KuduTable kuduTable = kuduClient.openTable(tableName);
|
||||||
//创建scanner扫描
|
// 创建scanner扫描
|
||||||
KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
|
KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
|
||||||
//遍历数据
|
// 遍历数据
|
||||||
while (scanner.hasMoreRows()) {
|
while (scanner.hasMoreRows()) {
|
||||||
while (scanner.nextRows().hasNext()) {
|
while (scanner.nextRows().hasNext()) {
|
||||||
rowCount++;
|
rowCount++;
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description 工具
|
* @description 工具
|
||||||
|
|||||||
@@ -1,27 +1,27 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.kudu.client.KuduClient;
|
import org.apache.kudu.client.KuduClient;
|
||||||
import org.apache.kudu.client.KuduException;
|
import org.apache.kudu.client.KuduException;
|
||||||
import org.apache.kudu.client.KuduTable;
|
import org.apache.kudu.client.KuduTable;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class KuduConnectionTest {
|
public class KuduConnectionTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void test01() {
|
public void test01() {
|
||||||
List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
|
List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
|
||||||
KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
|
KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
|
||||||
.defaultSocketReadTimeoutMs(30000).defaultAdminOperationTimeoutMs(60000).build();
|
.defaultSocketReadTimeoutMs(30000)
|
||||||
|
.defaultAdminOperationTimeoutMs(60000)
|
||||||
|
.build();
|
||||||
try {
|
try {
|
||||||
List<String> tablesList = kuduClient.getTablesList().getTablesList();
|
List<String> tablesList = kuduClient.getTablesList().getTablesList();
|
||||||
System.out.println(tablesList.toString());
|
System.out.println(tablesList.toString());
|
||||||
KuduTable web_white_list = kuduClient.openTable("web_white_list");
|
KuduTable web_white_list = kuduClient.openTable("web_white_list");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
} catch (KuduException e) {
|
} catch (KuduException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,36 +1,20 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import java.util.Map;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
|
||||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.Map;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ━━━━━━神兽出没━━━━━━
|
|
||||||
* ┏┓ ┏┓
|
|
||||||
* ┏┛┻━━━┛┻┓
|
|
||||||
* ┃ ━ ┃
|
|
||||||
* ┃ ┳┛ ┗┳ ┃
|
|
||||||
* ┃ ┻ ┃
|
|
||||||
* ┗━┓ ┏━┛
|
|
||||||
* ┃ ┃ 神兽保佑
|
|
||||||
* ┃ ┃ 代码无bug
|
|
||||||
* ┃ ┗━━━┓
|
|
||||||
* ┃ ┣┓
|
|
||||||
* ┃ ┏┛
|
|
||||||
* ┗┓┓┏━┳┓┏┛
|
|
||||||
* ┃┫┫ ┃┫┫
|
|
||||||
* ┗┻┛ ┗┻┛
|
|
||||||
* ━━━━━━感觉萌萌哒━━━━━━
|
|
||||||
* Created by Liuyadong on 2019-11-13
|
|
||||||
*
|
|
||||||
* @description
|
* @description
|
||||||
*/
|
*/
|
||||||
public class TestConfig {
|
public class TestConfig {
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void before() {
|
public void before() {
|
||||||
// 加载数据源连接池
|
// 加载数据源连接池
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
||||||
|
|
||||||
import com.alibaba.druid.pool.DruidDataSource;
|
|
||||||
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
|
|
||||||
|
import com.alibaba.druid.pool.DruidDataSource;
|
||||||
|
|
||||||
public class TestConstant {
|
public class TestConstant {
|
||||||
|
|
||||||
public final static String jdbcUrl = "jdbc:mysql://10.0.9.5:3306/canal_manager?useUnicode=true";
|
public final static String jdbcUrl = "jdbc:mysql://10.0.9.5:3306/canal_manager?useUnicode=true";
|
||||||
|
|||||||
@@ -1,35 +1,18 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.test.TestConstant;
|
import com.alibaba.otter.canal.client.adapter.kudu.test.TestConstant;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ━━━━━━神兽出没━━━━━━
|
|
||||||
* ┏┓ ┏┓
|
|
||||||
* ┏┛┻━━━┛┻┓
|
|
||||||
* ┃ ━ ┃
|
|
||||||
* ┃ ┳┛ ┗┳ ┃
|
|
||||||
* ┃ ┻ ┃
|
|
||||||
* ┗━┓ ┏━┛
|
|
||||||
* ┃ ┃ 神兽保佑
|
|
||||||
* ┃ ┃ 代码无bug
|
|
||||||
* ┃ ┗━━━┓
|
|
||||||
* ┃ ┣┓
|
|
||||||
* ┃ ┏┛
|
|
||||||
* ┗┓┓┏━┳┓┏┛
|
|
||||||
* ┃┫┫ ┃┫┫
|
|
||||||
* ┗┻┛ ┗┻┛
|
|
||||||
* ━━━━━━感觉萌萌哒━━━━━━
|
|
||||||
* Created by Liuyadong on 2019-11-13
|
|
||||||
*
|
|
||||||
* @description
|
* @description
|
||||||
*/
|
*/
|
||||||
public class Common {
|
public class Common {
|
||||||
|
|
||||||
public static KuduAdapter init() {
|
public static KuduAdapter init() {
|
||||||
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
|
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
|
||||||
|
|
||||||
|
|||||||
@@ -1,31 +1,19 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
import java.util.ArrayList;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
import java.util.Collections;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.*;
|
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ━━━━━━神兽出没━━━━━━
|
|
||||||
* ┏┓ ┏┓
|
|
||||||
* ┏┛┻━━━┛┻┓
|
|
||||||
* ┃ ━ ┃
|
|
||||||
* ┃ ┳┛ ┗┳ ┃
|
|
||||||
* ┃ ┻ ┃
|
|
||||||
* ┗━┓ ┏━┛
|
|
||||||
* ┃ ┃ 神兽保佑
|
|
||||||
* ┃ ┃ 代码无bug
|
|
||||||
* ┃ ┗━━━┓
|
|
||||||
* ┃ ┣┓
|
|
||||||
* ┃ ┏┛
|
|
||||||
* ┗┓┓┏━┳┓┏┛
|
|
||||||
* ┃┫┫ ┃┫┫
|
|
||||||
* ┗┻┛ ┗┻┛
|
|
||||||
* ━━━━━━感觉萌萌哒━━━━━━
|
|
||||||
* Created by Liuyadong on 2019-11-13
|
|
||||||
*
|
|
||||||
* @description
|
* @description
|
||||||
*/
|
*/
|
||||||
public class TestSyncKudu {
|
public class TestSyncKudu {
|
||||||
@@ -44,7 +32,7 @@ public class TestSyncKudu {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCount(){
|
public void testCount() {
|
||||||
kuduAdapter.count("kudutest_user.yml");
|
kuduAdapter.count("kudutest_user.yml");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user