mirror of
https://github.com/alibaba/canal.git
synced 2026-11-04 00:30:15 +08:00
fixed format
This commit is contained in:
@@ -1,5 +1,16 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu;
|
package com.alibaba.otter.canal.client.adapter.kudu;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
import com.alibaba.otter.canal.client.adapter.OuterAdapter;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
||||||
@@ -11,12 +22,6 @@ import com.alibaba.otter.canal.client.adapter.support.Dml;
|
|||||||
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
@@ -24,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
|
|||||||
*/
|
*/
|
||||||
@SPI("kudu")
|
@SPI("kudu")
|
||||||
public class KuduAdapter implements OuterAdapter {
|
public class KuduAdapter implements OuterAdapter {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(KuduAdapter.class);
|
private static Logger logger = LoggerFactory.getLogger(KuduAdapter.class);
|
||||||
|
|
||||||
private Map<String, KuduMappingConfig> kuduMapping = new ConcurrentHashMap<>(); // 文件名对应配置
|
private Map<String, KuduMappingConfig> kuduMapping = new ConcurrentHashMap<>(); // 文件名对应配置
|
||||||
@@ -47,7 +53,6 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
return mappingConfigCache;
|
return mappingConfigCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void init(OuterAdapterConfig configuration, Properties envProperties) {
|
public void init(OuterAdapterConfig configuration, Properties envProperties) {
|
||||||
this.envProperties = envProperties;
|
this.envProperties = envProperties;
|
||||||
@@ -72,12 +77,10 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
||||||
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
|
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-"
|
||||||
+ StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
|
+ StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_"
|
||||||
+ mappingConfig.getKuduMapping().getDatabase() + "-"
|
+ mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
|
||||||
+ mappingConfig.getKuduMapping().getTable();
|
|
||||||
} else {
|
} else {
|
||||||
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
|
k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_"
|
||||||
+ mappingConfig.getKuduMapping().getDatabase() + "-"
|
+ mappingConfig.getKuduMapping().getDatabase() + "-" + mappingConfig.getKuduMapping().getTable();
|
||||||
+ mappingConfig.getKuduMapping().getTable();
|
|
||||||
}
|
}
|
||||||
Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
|
Map<String, KuduMappingConfig> configMap = mappingConfigCache.computeIfAbsent(k,
|
||||||
k1 -> new ConcurrentHashMap<>());
|
k1 -> new ConcurrentHashMap<>());
|
||||||
@@ -108,9 +111,7 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
String table = dml.getTable();
|
String table = dml.getTable();
|
||||||
Map<String, KuduMappingConfig> configMap;
|
Map<String, KuduMappingConfig> configMap;
|
||||||
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
|
||||||
configMap = mappingConfigCache.get(destination + "-"
|
configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table);
|
||||||
+ groupId + "_"
|
|
||||||
+ database + "-" + table);
|
|
||||||
} else {
|
} else {
|
||||||
configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
|
configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
|
||||||
}
|
}
|
||||||
@@ -131,7 +132,8 @@ public class KuduAdapter implements OuterAdapter {
|
|||||||
logger.error("groupID didn't mach,please check your gruopId ");
|
logger.error("groupID didn't mach,please check your gruopId ");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-" + table);
|
logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-"
|
||||||
|
+ table);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +1,18 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description kudu配置文件映射
|
* @description kudu配置文件映射
|
||||||
*/
|
*/
|
||||||
public class KuduMappingConfig implements AdapterConfig {
|
public class KuduMappingConfig implements AdapterConfig {
|
||||||
|
|
||||||
private String dataSourceKey; // 数据源key
|
private String dataSourceKey; // 数据源key
|
||||||
|
|
||||||
private String destination; // canal实例或MQ的topic
|
private String destination; // canal实例或MQ的topic
|
||||||
@@ -77,7 +79,6 @@ public class KuduMappingConfig implements AdapterConfig {
|
|||||||
this.destination = destination;
|
this.destination = destination;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void validate() {
|
public void validate() {
|
||||||
if (kuduMapping.database == null || kuduMapping.database.isEmpty()) {
|
if (kuduMapping.database == null || kuduMapping.database.isEmpty()) {
|
||||||
throw new NullPointerException("KuduMapping.database");
|
throw new NullPointerException("KuduMapping.database");
|
||||||
|
|||||||
@@ -1,19 +1,21 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
package com.alibaba.otter.canal.client.adapter.kudu.config;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
|
||||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description kudu表信息加载
|
* @description kudu表信息加载
|
||||||
*/
|
*/
|
||||||
public class KuduMappingConfigLoader {
|
public class KuduMappingConfigLoader {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(KuduMappingConfigLoader.class);
|
private static Logger logger = LoggerFactory.getLogger(KuduMappingConfigLoader.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -28,8 +30,11 @@ public class KuduMappingConfigLoader {
|
|||||||
|
|
||||||
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
|
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("kudu");
|
||||||
configContentMap.forEach((fileName, content) -> {
|
configContentMap.forEach((fileName, content) -> {
|
||||||
KuduMappingConfig config = YmlConfigBinder
|
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||||
.bindYmlToObj(null, content, KuduMappingConfig.class, null, envProperties);
|
content,
|
||||||
|
KuduMappingConfig.class,
|
||||||
|
null,
|
||||||
|
envProperties);
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.monitor;
|
package com.alibaba.otter.canal.client.adapter.kudu.monitor;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
import java.io.File;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
import java.util.HashMap;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import java.util.Map;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
import java.util.Properties;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
|
||||||
import org.apache.commons.io.filefilter.FileFilterUtils;
|
import org.apache.commons.io.filefilter.FileFilterUtils;
|
||||||
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
|
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
|
||||||
import org.apache.commons.io.monitor.FileAlterationMonitor;
|
import org.apache.commons.io.monitor.FileAlterationMonitor;
|
||||||
@@ -13,16 +13,18 @@ import org.apache.commons.lang.StringUtils;
|
|||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.File;
|
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
|
||||||
import java.util.HashMap;
|
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||||
import java.util.Map;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import java.util.Properties;
|
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description 配置文件监听
|
* @description 配置文件监听
|
||||||
*/
|
*/
|
||||||
public class KuduConfigMonitor {
|
public class KuduConfigMonitor {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(KuduConfigMonitor.class);
|
private static final Logger logger = LoggerFactory.getLogger(KuduConfigMonitor.class);
|
||||||
|
|
||||||
private static final String adapterName = "kudu";
|
private static final String adapterName = "kudu";
|
||||||
@@ -64,6 +66,7 @@ public class KuduConfigMonitor {
|
|||||||
* 配置文件监听
|
* 配置文件监听
|
||||||
*/
|
*/
|
||||||
private class FileListener extends FileAlterationListenerAdaptor {
|
private class FileListener extends FileAlterationListenerAdaptor {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onFileCreate(File file) {
|
public void onFileCreate(File file) {
|
||||||
super.onFileCreate(file);
|
super.onFileCreate(file);
|
||||||
@@ -71,8 +74,11 @@ public class KuduConfigMonitor {
|
|||||||
try {
|
try {
|
||||||
// 加载新增的配置文件
|
// 加载新增的配置文件
|
||||||
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
|
||||||
KuduMappingConfig config = YmlConfigBinder
|
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||||
.bindYmlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
|
configContent,
|
||||||
|
KuduMappingConfig.class,
|
||||||
|
null,
|
||||||
|
envProperties);
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -92,14 +98,17 @@ public class KuduConfigMonitor {
|
|||||||
try {
|
try {
|
||||||
if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
|
if (kuduAdapter.getKuduMapping().containsKey(file.getName())) {
|
||||||
// 加载配置文件
|
// 加载配置文件
|
||||||
String configContent = MappingConfigsLoader
|
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
|
||||||
.loadConfig(adapterName + File.separator + file.getName());
|
+ file.getName());
|
||||||
if (configContent == null) {
|
if (configContent == null) {
|
||||||
onFileDelete(file);
|
onFileDelete(file);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
KuduMappingConfig config = YmlConfigBinder
|
KuduMappingConfig config = YmlConfigBinder.bindYmlToObj(null,
|
||||||
.bindYmlToObj(null, configContent, KuduMappingConfig.class, null, envProperties);
|
configContent,
|
||||||
|
KuduMappingConfig.class,
|
||||||
|
null,
|
||||||
|
envProperties);
|
||||||
if (config == null) {
|
if (config == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -114,7 +123,6 @@ public class KuduConfigMonitor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onFileDelete(File file) {
|
public void onFileDelete(File file) {
|
||||||
super.onFileDelete(file);
|
super.onFileDelete(file);
|
||||||
|
|||||||
@@ -1,5 +1,18 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
||||||
|
|
||||||
|
import java.sql.ResultSetMetaData;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
|
||||||
|
import org.apache.kudu.client.KuduException;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.support.SyncUtil;
|
import com.alibaba.otter.canal.client.adapter.kudu.support.SyncUtil;
|
||||||
@@ -8,13 +21,6 @@ import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
|
|||||||
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Util;
|
import com.alibaba.otter.canal.client.adapter.support.Util;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
import org.apache.kudu.client.KuduException;
|
|
||||||
|
|
||||||
import javax.sql.DataSource;
|
|
||||||
import java.sql.ResultSetMetaData;
|
|
||||||
import java.sql.SQLException;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
@@ -49,9 +55,9 @@ public class KuduEtlService extends AbstractEtlService {
|
|||||||
return importData(sql, params);
|
return importData(sql, params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values, AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
|
protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
|
||||||
|
AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
|
||||||
KuduMappingConfig.KuduMapping kuduMapping = (KuduMappingConfig.KuduMapping) mapping;
|
KuduMappingConfig.KuduMapping kuduMapping = (KuduMappingConfig.KuduMapping) mapping;
|
||||||
// 获取字段元数据
|
// 获取字段元数据
|
||||||
Map<String, String> columnsMap = new LinkedHashMap<>();// 需要同步的字段
|
Map<String, String> columnsMap = new LinkedHashMap<>();// 需要同步的字段
|
||||||
|
|||||||
@@ -1,28 +1,34 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
package com.alibaba.otter.canal.client.adapter.kudu.service;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.kudu.client.KuduException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.JSON;
|
||||||
import com.alibaba.fastjson.serializer.SerializerFeature;
|
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
import com.alibaba.otter.canal.client.adapter.kudu.support.KuduTemplate;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||||
import org.apache.kudu.client.KuduException;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description kudu实时同步
|
* @description kudu实时同步
|
||||||
*/
|
*/
|
||||||
public class KuduSyncService {
|
public class KuduSyncService {
|
||||||
|
|
||||||
private static Logger logger = LoggerFactory.getLogger(KuduSyncService.class);
|
private static Logger logger = LoggerFactory.getLogger(KuduSyncService.class);
|
||||||
|
|
||||||
private KuduTemplate kuduTemplate;
|
private KuduTemplate kuduTemplate;
|
||||||
|
|
||||||
// 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
|
// 源库表字段类型缓存: instance.schema.table -> <columnName, jdbcType>
|
||||||
// private Map<String, Map<String, Integer>> columnsTypeCache = new ConcurrentHashMap<>();
|
// private Map<String, Map<String, Integer>> columnsTypeCache = new
|
||||||
|
// ConcurrentHashMap<>();
|
||||||
|
|
||||||
public KuduSyncService(KuduTemplate kuduTemplate){
|
public KuduSyncService(KuduTemplate kuduTemplate){
|
||||||
this.kuduTemplate = kuduTemplate;
|
this.kuduTemplate = kuduTemplate;
|
||||||
@@ -157,7 +163,6 @@ public class KuduSyncService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,16 +1,30 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
||||||
|
|
||||||
import org.apache.kudu.ColumnSchema;
|
|
||||||
import org.apache.kudu.Schema;
|
|
||||||
import org.apache.kudu.Type;
|
|
||||||
import org.apache.kudu.client.*;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.text.ParseException;
|
import java.text.ParseException;
|
||||||
import java.text.SimpleDateFormat;
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.*;
|
import java.util.Arrays;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.kudu.ColumnSchema;
|
||||||
|
import org.apache.kudu.Schema;
|
||||||
|
import org.apache.kudu.Type;
|
||||||
|
import org.apache.kudu.client.Delete;
|
||||||
|
import org.apache.kudu.client.Insert;
|
||||||
|
import org.apache.kudu.client.KuduClient;
|
||||||
|
import org.apache.kudu.client.KuduException;
|
||||||
|
import org.apache.kudu.client.KuduScanner;
|
||||||
|
import org.apache.kudu.client.KuduSession;
|
||||||
|
import org.apache.kudu.client.KuduTable;
|
||||||
|
import org.apache.kudu.client.OperationResponse;
|
||||||
|
import org.apache.kudu.client.PartialRow;
|
||||||
|
import org.apache.kudu.client.SessionConfiguration;
|
||||||
|
import org.apache.kudu.client.Upsert;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
@@ -39,10 +53,10 @@ public class KuduTemplate {
|
|||||||
if (kuduClient == null) {
|
if (kuduClient == null) {
|
||||||
// kudu master 以逗号分隔
|
// kudu master 以逗号分隔
|
||||||
List<String> masterList = Arrays.asList(masters.split(","));
|
List<String> masterList = Arrays.asList(masters.split(","));
|
||||||
kuduClient = new KuduClient.KuduClientBuilder(masterList)
|
kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
|
||||||
.defaultOperationTimeoutMs(60000)
|
|
||||||
.defaultSocketReadTimeoutMs(60000)
|
.defaultSocketReadTimeoutMs(60000)
|
||||||
.defaultAdminOperationTimeoutMs(60000).build();
|
.defaultAdminOperationTimeoutMs(60000)
|
||||||
|
.build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,7 +206,6 @@ public class KuduTemplate {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -207,7 +220,8 @@ public class KuduTemplate {
|
|||||||
KuduTable kuduTable = kuduClient.openTable(tableName);// 打开表
|
KuduTable kuduTable = kuduClient.openTable(tableName);// 打开表
|
||||||
KuduSession session = kuduClient.newSession(); // 创建写session,kudu必须通过session写入
|
KuduSession session = kuduClient.newSession(); // 创建写session,kudu必须通过session写入
|
||||||
try {
|
try {
|
||||||
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // 采取Flush方式 手动刷新
|
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); // 采取Flush方式
|
||||||
|
// 手动刷新
|
||||||
session.setMutationBufferSpace(OPERATION_BATCH);
|
session.setMutationBufferSpace(OPERATION_BATCH);
|
||||||
// 获取元数据结构
|
// 获取元数据结构
|
||||||
Map<String, Type> metaMap = new HashMap<>();
|
Map<String, Type> metaMap = new HashMap<>();
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
package com.alibaba.otter.canal.client.adapter.kudu.support;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @author liuyadong
|
* @author liuyadong
|
||||||
* @description 工具
|
* @description 工具
|
||||||
|
|||||||
@@ -1,27 +1,27 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.kudu.client.KuduClient;
|
import org.apache.kudu.client.KuduClient;
|
||||||
import org.apache.kudu.client.KuduException;
|
import org.apache.kudu.client.KuduException;
|
||||||
import org.apache.kudu.client.KuduTable;
|
import org.apache.kudu.client.KuduTable;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class KuduConnectionTest {
|
public class KuduConnectionTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void test01() {
|
public void test01() {
|
||||||
List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
|
List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
|
||||||
KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
|
KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
|
||||||
.defaultSocketReadTimeoutMs(30000).defaultAdminOperationTimeoutMs(60000).build();
|
.defaultSocketReadTimeoutMs(30000)
|
||||||
|
.defaultAdminOperationTimeoutMs(60000)
|
||||||
|
.build();
|
||||||
try {
|
try {
|
||||||
List<String> tablesList = kuduClient.getTablesList().getTablesList();
|
List<String> tablesList = kuduClient.getTablesList().getTablesList();
|
||||||
System.out.println(tablesList.toString());
|
System.out.println(tablesList.toString());
|
||||||
KuduTable web_white_list = kuduClient.openTable("web_white_list");
|
KuduTable web_white_list = kuduClient.openTable("web_white_list");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
} catch (KuduException e) {
|
} catch (KuduException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,36 +1,20 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
import java.util.Map;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
|
||||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.Map;
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfig;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.kudu.config.KuduMappingConfigLoader;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ━━━━━━神兽出没━━━━━━
|
|
||||||
* ┏┓ ┏┓
|
|
||||||
* ┏┛┻━━━┛┻┓
|
|
||||||
* ┃ ━ ┃
|
|
||||||
* ┃ ┳┛ ┗┳ ┃
|
|
||||||
* ┃ ┻ ┃
|
|
||||||
* ┗━┓ ┏━┛
|
|
||||||
* ┃ ┃ 神兽保佑
|
|
||||||
* ┃ ┃ 代码无bug
|
|
||||||
* ┃ ┗━━━┓
|
|
||||||
* ┃ ┣┓
|
|
||||||
* ┃ ┏┛
|
|
||||||
* ┗┓┓┏━┳┓┏┛
|
|
||||||
* ┃┫┫ ┃┫┫
|
|
||||||
* ┗┻┛ ┗┻┛
|
|
||||||
* ━━━━━━感觉萌萌哒━━━━━━
|
|
||||||
* Created by Liuyadong on 2019-11-13
|
|
||||||
*
|
|
||||||
* @description
|
* @description
|
||||||
*/
|
*/
|
||||||
public class TestConfig {
|
public class TestConfig {
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void before() {
|
public void before() {
|
||||||
// 加载数据源连接池
|
// 加载数据源连接池
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
package com.alibaba.otter.canal.client.adapter.kudu.test;
|
||||||
|
|
||||||
import com.alibaba.druid.pool.DruidDataSource;
|
|
||||||
|
|
||||||
import java.sql.SQLException;
|
import java.sql.SQLException;
|
||||||
|
|
||||||
|
import com.alibaba.druid.pool.DruidDataSource;
|
||||||
|
|
||||||
public class TestConstant {
|
public class TestConstant {
|
||||||
|
|
||||||
public final static String jdbcUrl = "jdbc:mysql://10.0.9.5:3306/canal_manager?useUnicode=true";
|
public final static String jdbcUrl = "jdbc:mysql://10.0.9.5:3306/canal_manager?useUnicode=true";
|
||||||
|
|||||||
@@ -1,35 +1,18 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.test.TestConstant;
|
import com.alibaba.otter.canal.client.adapter.kudu.test.TestConstant;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ━━━━━━神兽出没━━━━━━
|
|
||||||
* ┏┓ ┏┓
|
|
||||||
* ┏┛┻━━━┛┻┓
|
|
||||||
* ┃ ━ ┃
|
|
||||||
* ┃ ┳┛ ┗┳ ┃
|
|
||||||
* ┃ ┻ ┃
|
|
||||||
* ┗━┓ ┏━┛
|
|
||||||
* ┃ ┃ 神兽保佑
|
|
||||||
* ┃ ┃ 代码无bug
|
|
||||||
* ┃ ┗━━━┓
|
|
||||||
* ┃ ┣┓
|
|
||||||
* ┃ ┏┛
|
|
||||||
* ┗┓┓┏━┳┓┏┛
|
|
||||||
* ┃┫┫ ┃┫┫
|
|
||||||
* ┗┻┛ ┗┻┛
|
|
||||||
* ━━━━━━感觉萌萌哒━━━━━━
|
|
||||||
* Created by Liuyadong on 2019-11-13
|
|
||||||
*
|
|
||||||
* @description
|
* @description
|
||||||
*/
|
*/
|
||||||
public class Common {
|
public class Common {
|
||||||
|
|
||||||
public static KuduAdapter init() {
|
public static KuduAdapter init() {
|
||||||
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
|
DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
|
||||||
|
|
||||||
|
|||||||
@@ -1,31 +1,19 @@
|
|||||||
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
package com.alibaba.otter.canal.client.adapter.kudu.test.sync;
|
||||||
|
|
||||||
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
import java.util.ArrayList;
|
||||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
import java.util.Collections;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.util.*;
|
import com.alibaba.otter.canal.client.adapter.kudu.KuduAdapter;
|
||||||
|
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ━━━━━━神兽出没━━━━━━
|
|
||||||
* ┏┓ ┏┓
|
|
||||||
* ┏┛┻━━━┛┻┓
|
|
||||||
* ┃ ━ ┃
|
|
||||||
* ┃ ┳┛ ┗┳ ┃
|
|
||||||
* ┃ ┻ ┃
|
|
||||||
* ┗━┓ ┏━┛
|
|
||||||
* ┃ ┃ 神兽保佑
|
|
||||||
* ┃ ┃ 代码无bug
|
|
||||||
* ┃ ┗━━━┓
|
|
||||||
* ┃ ┣┓
|
|
||||||
* ┃ ┏┛
|
|
||||||
* ┗┓┓┏━┳┓┏┛
|
|
||||||
* ┃┫┫ ┃┫┫
|
|
||||||
* ┗┻┛ ┗┻┛
|
|
||||||
* ━━━━━━感觉萌萌哒━━━━━━
|
|
||||||
* Created by Liuyadong on 2019-11-13
|
|
||||||
*
|
|
||||||
* @description
|
* @description
|
||||||
*/
|
*/
|
||||||
public class TestSyncKudu {
|
public class TestSyncKudu {
|
||||||
|
|||||||
Reference in New Issue
Block a user