chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #849: hbase数据同步适配

Browse Source
This commit is contained in:
mcy
2018-08-23 12:02:44 +08:00
parent 47348e2e96
commit 20b0b39b14
39 changed files with 3182 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java Normal file
View File

@@ -0,0 +1,34 @@
package com.alibaba.otter.canal.client.adapter;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.protocol.Message;
/**
* 外部适配器接口
*
* @author machengyuan 2018-8-18 下午10:14:02
* @version 1.0.0
*/
@SPI("logger")
public interface CanalOuterAdapter {
/**
* 外部适配器初始化接口
*
* @param configuration 外部适配器配置信息
*/
void init(CanalOuterAdapterConfiguration configuration);
/**
* 往适配器中写入数据
*
* @param message message数据包
*/
void writeOut(Message message);
/**
* 外部适配器销毁接口
*/
void destroy();
}

181
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java Normal file
View File

@@ -0,0 +1,181 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
/**
* 配置信息类
*
* @author machengyuan 2018-8-18 下午10:40:12
* @version 1.0.0
*/
public class CanalClientConfig {
private String canalServerHost;
private String zookeeperHosts;
private Properties properties;
private String bootstrapServers;
private List<KafkaTopic> kafkaTopics = new ArrayList<>();
private List<CanalInstance> canalInstances = new ArrayList<>();
public String getCanalServerHost() {
return canalServerHost;
}
public void setCanalServerHost(String canalServerHost) {
this.canalServerHost = canalServerHost;
}
public String getZookeeperHosts() {
return zookeeperHosts;
}
public void setZookeeperHosts(String zookeeperHosts) {
this.zookeeperHosts = zookeeperHosts;
}
public Properties getProperties() {
return properties;
}
public void setProperties(Properties properties) {
this.properties = properties;
}
public String getBootstrapServers() {
return bootstrapServers;
}
public void setBootstrapServers(String bootstrapServers) {
this.bootstrapServers = bootstrapServers;
}
public List<KafkaTopic> getKafkaTopics() {
return kafkaTopics;
}
public void setKafkaTopics(List<KafkaTopic> kafkaTopics) {
this.kafkaTopics = kafkaTopics;
}
public List<CanalInstance> getCanalInstances() {
return canalInstances;
}
public void setCanalInstances(List<CanalInstance> canalInstances) {
this.canalInstances = canalInstances;
}
public static class CanalInstance {
private String instance;
private List<AdapterGroup> adapterGroups;
public String getInstance() {
return instance;
}
public void setInstance(String instance) {
if (instance != null) {
this.instance = instance.trim();
}
}
public List<AdapterGroup> getAdapterGroups() {
return adapterGroups;
}
public void setAdapterGroups(List<AdapterGroup> adapterGroups) {
this.adapterGroups = adapterGroups;
}
}
public static class AdapterGroup {
private List<CanalOuterAdapterConfiguration> outAdapters;
public List<CanalOuterAdapterConfiguration> getOutAdapters() {
return outAdapters;
}
public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters) {
this.outAdapters = outAdapters;
}
}
public static class KafkaTopic {
private String topic;
private List<Group> groups = new ArrayList<>();
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public List<Group> getGroups() {
return groups;
}
public void setGroups(List<Group> groups) {
this.groups = groups;
}
}
public static class Group {
private String groupId;
// private List<Adaptor> adapters = new ArrayList<>();
private List<CanalOuterAdapterConfiguration> outAdapters;
public String getGroupId() {
return groupId;
}
public void setGroupId(String groupId) {
this.groupId = groupId;
}
public List<CanalOuterAdapterConfiguration> getOutAdapters() {
return outAdapters;
}
public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters) {
this.outAdapters = outAdapters;
}
// public List<Adaptor> getAdapters() {
// return adapters;
// }
//
// public void setAdapters(List<Adaptor> adapters) {
// this.adapters = adapters;
// }
}
// public static class Adaptor {
// private List<CanalOuterAdapterConfiguration> outAdapters;
//
// public List<CanalOuterAdapterConfiguration> getOutAdapters() {
// return outAdapters;
// }
//
// public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters)
// {
// this.outAdapters = outAdapters;
// }
// }
}

52
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalOuterAdapterConfiguration.java Normal file
View File

@@ -0,0 +1,52 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.util.Properties;
/**
* 外部适配器配置信息类
*
* @author machengyuan 2018-8-18 下午10:15:12
* @version 1.0.0
*/
public class CanalOuterAdapterConfiguration {
private String name; // 适配器名称, 如: logger, hbase, es
private String hosts; // 适配器内部的地址, 比如对应es该参数可以填写es的server地址
private String zkHosts; // 适配器内部的ZK地址, 比如对应HBase该参数可以填写HBase对应的ZK地址
private Properties properties; // 其余参数, 可填写适配器中的所需的配置信息
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getHosts() {
return hosts;
}
public void setHosts(String hosts) {
this.hosts = hosts;
}
public Properties getProperties() {
return properties;
}
public void setProperties(Properties properties) {
this.properties = properties;
}
public String getZkHosts() {
return zkHosts;
}
public void setZkHosts(String zkHosts) {
this.zkHosts = zkHosts;
}
}

96
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java Normal file
View File

@@ -0,0 +1,96 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
/**
* DML操作转换对象
*
* @author machengyuan 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class Dml implements Serializable {
private static final long serialVersionUID = 2611556444074013268L;
private String database;
private String table;
private String type;
private Long ts;
private String sql;
private List<Map<String, Object>> data;
private List<Map<String, Object>> old;
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Long getTs() {
return ts;
}
public void setTs(Long ts) {
this.ts = ts;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<Map<String, Object>> getData() {
return data;
}
public void setData(List<Map<String, Object>> data) {
this.data = data;
}
public List<Map<String, Object>> getOld() {
return old;
}
public void setOld(List<Map<String, Object>> old) {
this.old = old;
}
public void clear() {
database = null;
table = null;
type = null;
ts = null;
data = null;
old = null;
sql = null;
}
@Override
public String toString() {
return "Dml{" + "database='" + database + '\'' + ", table='" + table + '\'' + ", type='" + type + '\'' + ", ts="
+ ts + ", sql='" + sql + '\'' + ", data=" + data + ", old=" + old + '}';
}
}

563
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java Normal file
View File

@@ -0,0 +1,563 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SPI 类加载器
*
* @author machengyuan 2018-8-19 下午11:30:49
* @version 1.0.0
*/
public class ExtensionLoader<T> {
private static final Logger logger = LoggerFactory
.getLogger(ExtensionLoader.class);
private static final String SERVICES_DIRECTORY = "META-INF/services/";
private static final String CANAL_DIRECTORY = "META-INF/canal/";
private static final String DEFAULT_CLASSLOADER_POLICY = "internal";
private static final Pattern NAME_SEPARATOR = Pattern
.compile("\\s*[,]+\\s*");
private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS = new ConcurrentHashMap<>();
private static final ConcurrentMap<Class<?>, Object> EXTENSION_INSTANCES = new ConcurrentHashMap<>();
private final Class<?> type;
private final String classLoaderPolicy;
private final ConcurrentMap<Class<?>, String> cachedNames = new ConcurrentHashMap<>();
private final Holder<Map<String, Class<?>>> cachedClasses = new Holder<>();
private final ConcurrentMap<String, Holder<Object>> cachedInstances = new ConcurrentHashMap<>();
private String cachedDefaultName;
private ConcurrentHashMap<String, IllegalStateException> exceptions = new ConcurrentHashMap<>();
private static <T> boolean withExtensionAnnotation(Class<T> type) {
return type.isAnnotationPresent(SPI.class);
}
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
}
@SuppressWarnings("unchecked")
public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
if (type == null) throw new IllegalArgumentException("Extension type == null");
if (!type.isInterface()) {
throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
}
if (!withExtensionAnnotation(type)) {
throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+ SPI.class.getSimpleName() + " Annotation!");
}
ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
if (loader == null) {
EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
}
return loader;
}
private ExtensionLoader(Class<?> type, String classLoaderPolicy){
this.type = type;
this.classLoaderPolicy = classLoaderPolicy;
}
public String getExtensionName(T extensionInstance) {
return getExtensionName(extensionInstance.getClass());
}
public String getExtensionName(Class<?> extensionClass) {
return cachedNames.get(extensionClass);
}
public ConcurrentHashMap<String, IllegalStateException> getExceptions() {
return exceptions;
}
/**
* 返回扩展点实例，如果没有指定的扩展点或是还没加载（即实例化）则返回<code>null</code>注意：此方法不会触发扩展点的加载
* <p/>
* 一般应该调用{@link #getExtension(String)}方法获得扩展，这个方法会触发扩展点加载
*
* @see #getExtension(String)
*/
@SuppressWarnings("unchecked")
public T getLoadedExtension(String name) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
Holder<Object> holder = cachedInstances.get(name);
if (holder == null) {
cachedInstances.putIfAbsent(name, new Holder<>());
holder = cachedInstances.get(name);
}
return (T) holder.get();
}
/**
* 返回已经加载的扩展点的名字
* <p/>
* 一般应该调用{@link #getSupportedExtensions()}方法获得扩展，这个方法会返回所有的扩展点
*
* @see #getSupportedExtensions()
*/
public Set<String> getLoadedExtensions() {
return Collections.unmodifiableSet(new TreeSet<>(cachedInstances.keySet()));
}
/**
* 返回指定名字的扩展
*
* @param name
* @return
*/
@SuppressWarnings("unchecked")
public T getExtension(String name) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
if ("true".equals(name)) {
return getDefaultExtension();
}
Holder<Object> holder = cachedInstances.get(name);
if (holder == null) {
cachedInstances.putIfAbsent(name, new Holder<>());
holder = cachedInstances.get(name);
}
Object instance = holder.get();
if (instance == null) {
synchronized (holder) {
instance = holder.get();
if (instance == null) {
instance = createExtension(name);
holder.set(instance);
}
}
}
return (T) instance;
}
/**
* 返回缺省的扩展，如果没有设置则返回<code>null</code>
*/
public T getDefaultExtension() {
getExtensionClasses();
if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
return null;
}
return getExtension(cachedDefaultName);
}
public boolean hasExtension(String name) {
if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
try {
return getExtensionClass(name) != null;
} catch (Throwable t) {
return false;
}
}
public Set<String> getSupportedExtensions() {
Map<String, Class<?>> clazzes = getExtensionClasses();
return Collections.unmodifiableSet(new TreeSet<String>(clazzes.keySet()));
}
/**
* 返回缺省的扩展点名，如果没有设置缺省则返回<code>null</code>
*/
public String getDefaultExtensionName() {
getExtensionClasses();
return cachedDefaultName;
}
/**
* 编程方式添加新扩展点
*
* @param name 扩展点名
* @param clazz 扩展点类
* @throws IllegalStateException 要添加扩展点名已经存在
*/
public void addExtension(String name, Class<?> clazz) {
getExtensionClasses(); // load classes
if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException("Input type " + clazz + "not implement Extension " + type);
}
if (clazz.isInterface()) {
throw new IllegalStateException("Input type " + clazz + "can not be interface!");
}
if (name == null || "".equals(name)) {
throw new IllegalStateException("Extension name is blank (Extension " + type + ")!");
}
if (cachedClasses.get().containsKey(name)) {
throw new IllegalStateException("Extension name " + name + " already existed(Extension " + type + ")!");
}
cachedNames.put(clazz, name);
cachedClasses.get().put(name, clazz);
}
/**
* 编程方式添加替换已有扩展点
*
* @param name 扩展点名
* @param clazz 扩展点类
* @throws IllegalStateException 要添加扩展点名已经存在
* @deprecated 不推荐应用使用，一般只在测试时可以使用
*/
@Deprecated
public void replaceExtension(String name, Class<?> clazz) {
getExtensionClasses(); // load classes
if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException("Input type " + clazz + "not implement Extension " + type);
}
if (clazz.isInterface()) {
throw new IllegalStateException("Input type " + clazz + "can not be interface!");
}
if (name == null || "".equals(name)) {
throw new IllegalStateException("Extension name is blank (Extension " + type + ")!");
}
if (!cachedClasses.get().containsKey(name)) {
throw new IllegalStateException("Extension name " + name + " not existed(Extension " + type + ")!");
}
cachedNames.put(clazz, name);
cachedClasses.get().put(name, clazz);
cachedInstances.remove(name);
}
@SuppressWarnings("unchecked")
private T createExtension(String name) {
Class<?> clazz = getExtensionClasses().get(name);
if (clazz == null) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: class could not be found");
}
try {
T instance = (T) EXTENSION_INSTANCES.get(clazz);
if (instance == null) {
EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
instance = (T) EXTENSION_INSTANCES.get(clazz);
}
return instance;
} catch (Throwable t) {
throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+ ") could not be instantiated: " + t.getMessage(),
t);
}
}
private Class<?> getExtensionClass(String name) {
if (type == null) throw new IllegalArgumentException("Extension type == null");
if (name == null) throw new IllegalArgumentException("Extension name == null");
Class<?> clazz = getExtensionClasses().get(name);
if (clazz == null)
throw new IllegalStateException("No such extension \"" + name + "\" for " + type.getName() + "!");
return clazz;
}
private Map<String, Class<?>> getExtensionClasses() {
Map<String, Class<?>> classes = cachedClasses.get();
if (classes == null) {
synchronized (cachedClasses) {
classes = cachedClasses.get();
if (classes == null) {
classes = loadExtensionClasses();
cachedClasses.set(classes);
}
}
}
return classes;
}
private String getJarDirectoryPath() {
URL url = Thread.currentThread().getContextClassLoader().getResource("");
if (url == null) {
throw new IllegalStateException("failed to get class loader resource");
}
String dirtyPath = url.toString();
String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes file:/ and everything before it
jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything after .jar, if .jar exists in dirtyPath
jarPath = jarPath.replaceAll("%20", " "); // necessary if path has spaces within
if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run the app using Spring Tools Suit play
// button.
jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
}
return Paths.get(jarPath).getParent().toString(); // Paths - from java 8
}
private Map<String, Class<?>> loadExtensionClasses() {
final SPI defaultAnnotation = type.getAnnotation(SPI.class);
if (defaultAnnotation != null) {
String value = defaultAnnotation.value();
if ((value = value.trim()).length() > 0) {
String[] names = NAME_SEPARATOR.split(value);
if (names.length > 1) {
throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+ ": " + Arrays.toString(names));
}
if (names.length == 1) cachedDefaultName = names[0];
}
}
Map<String, Class<?>> extensionClasses = new HashMap<String, Class<?>>();
// 1. lib folder，customized extension classLoader （jar_dir/lib）
String dir = File.separator + this.getJarDirectoryPath() + File.separator + "lib";
logger.info("extension classpath dir: " + dir);
File externalLibDir = new File(dir);
if (!externalLibDir.exists()) {
externalLibDir = new File(
File.separator + this.getJarDirectoryPath() + File.separator + "canal_client" + File.separator + "lib");
}
if (externalLibDir.exists()) {
File[] files = externalLibDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".jar");
}
});
if (files != null) {
for (File f : files) {
URL url = null;
try {
url = f.toURI().toURL();
} catch (MalformedURLException e) {
throw new RuntimeException("load extension jar failed!", e);
}
ClassLoader parent = Thread.currentThread().getContextClassLoader();
URLClassLoader localClassLoader;
if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
|| DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
localClassLoader = new URLClassLoader(new URL[] { url }, parent) {
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
Class<?> c = findLoadedClass(name);
if (c != null) {
return c;
}
if (name.startsWith("java.") || name.startsWith("org.slf4j.")
|| name.startsWith("org.apache.logging")
|| name.startsWith("org.apache.commons.logging.")) {
// || name.startsWith("org.apache.hadoop.")) {
c = super.loadClass(name);
}
if (c != null) return c;
try {
// 先加载jar内的class，可避免jar冲突
c = findClass(name);
} catch (ClassNotFoundException e) {
c = null;
}
if (c != null) {
return c;
}
return super.loadClass(name);
}
@Override
public Enumeration<URL> getResources(String name) throws IOException {
@SuppressWarnings("unchecked")
Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
tmp[0] = findResources(name); // local class path first
// tmp[1] = super.getResources(name);
return new CompoundEnumeration<>(tmp);
}
};
} else {
localClassLoader = new URLClassLoader(new URL[] { url }, parent);
}
loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
}
}
}
// 2. load inner extension class with default classLoader
ClassLoader classLoader = findClassLoader();
loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
return extensionClasses;
}
public static class CompoundEnumeration<E> implements Enumeration<E> {
private Enumeration<E>[] enums;
private int index = 0;
public CompoundEnumeration(Enumeration<E>[] enums){
this.enums = enums;
}
private boolean next() {
while (this.index < this.enums.length) {
if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
return true;
}
++this.index;
}
return false;
}
public boolean hasMoreElements() {
return this.next();
}
public E nextElement() {
if (!this.next()) {
throw new NoSuchElementException();
} else {
return this.enums[this.index].nextElement();
}
}
}
private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
String fileName = dir + type.getName();
try {
Enumeration<URL> urls;
if (classLoader != null) {
urls = classLoader.getResources(fileName);
} else {
urls = ClassLoader.getSystemResources(fileName);
}
if (urls != null) {
while (urls.hasMoreElements()) {
URL url = urls.nextElement();
try {
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));
String line = null;
while ((line = reader.readLine()) != null) {
final int ci = line.indexOf('#');
if (ci >= 0) line = line.substring(0, ci);
line = line.trim();
if (line.length() > 0) {
try {
String name = null;
int i = line.indexOf('=');
if (i > 0) {
name = line.substring(0, i).trim();
line = line.substring(i + 1).trim();
}
if (line.length() > 0) {
Class<?> clazz = classLoader.loadClass(line);
// Class<?> clazz = Class.forName(line, true, classLoader);
if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException(
"Error when load extension class(interface: " + type
+ ", class line: " + clazz.getName()
+ "), class " + clazz.getName()
+ "is not subtype of interface.");
} else {
try {
clazz.getConstructor(type);
} catch (NoSuchMethodException e) {
clazz.getConstructor();
String[] names = NAME_SEPARATOR.split(name);
if (names != null && names.length > 0) {
for (String n : names) {
if (!cachedNames.containsKey(clazz)) {
cachedNames.put(clazz, n);
}
Class<?> c = extensionClasses.get(n);
if (c == null) {
extensionClasses.put(n, clazz);
} else if (c != clazz) {
cachedNames.remove(clazz);
throw new IllegalStateException(
"Duplicate extension " + type.getName() + " name "
+ n + " on "
+ c.getName() + " and "
+ clazz.getName());
}
}
}
}
}
}
} catch (Throwable t) {
IllegalStateException e = new IllegalStateException(
"Failed to load extension class(interface: " + type + ", class line: "
+ line + ") in " + url
+ ", cause: "
+ t.getMessage(),
t);
exceptions.put(line, e);
}
}
} // end of while read lines
} finally {
if (reader != null) {
reader.close();
}
}
} catch (Throwable t) {
logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+ ") in " + url,
t);
}
} // end of while urls
}
} catch (Throwable t) {
logger.error(
"Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
t);
}
}
private static ClassLoader findClassLoader() {
return ExtensionLoader.class.getClassLoader();
}
@Override
public String toString() {
return this.getClass().getName() + "[" + type.getName() + "]";
}
private static class Holder<T> {
private volatile T value;
private void set(T value) {
this.value = value;
}
private T get() {
return value;
}
}
}

103
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java Normal file
View File

@@ -0,0 +1,103 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 类型转换工具类
*
* @author machengyuan 2018-8-19 下午06:14:23
* @version 1.0.0
*/
public class JdbcTypeUtil {
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
if (value == null || value.equals("")) {
return null;
}
try {
Object res;
switch (sqlType) {
case Types.INTEGER:
res = Integer.parseInt(value);
break;
case Types.SMALLINT:
res = Short.parseShort(value);
break;
case Types.BIT:
case Types.TINYINT:
res = Byte.parseByte(value);
break;
case Types.BIGINT:
if (mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
res = new BigInteger(value);
} else {
res = Long.parseLong(value);
}
break;
// case Types.BIT:
case Types.BOOLEAN:
res = !"0".equals(value);
break;
case Types.DOUBLE:
case Types.FLOAT:
res = Double.parseDouble(value);
break;
case Types.REAL:
res = Float.parseFloat(value);
break;
case Types.DECIMAL:
case Types.NUMERIC:
res = new BigDecimal(value);
break;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
case Types.BLOB:
res = value.getBytes("ISO-8859-1");
break;
case Types.DATE:
if (!value.startsWith("0000-00-00")) {
value = value.trim().replace(" ", "T");
DateTime dt = new DateTime(value);
res = new Date(dt.toDate().getTime());
} else {
res = null;
}
break;
case Types.TIME:
value = "T" + value;
DateTime dt = new DateTime(value);
res = new Time(dt.toDate().getTime());
break;
case Types.TIMESTAMP:
if (!value.startsWith("0000-00-00")) {
value = value.trim().replace(" ", "T");
dt = new DateTime(value);
res = new Timestamp(dt.toDate().getTime());
} else {
res = null;
}
break;
case Types.CLOB:
default:
res = value;
}
return res;
} catch (Exception e) {
logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
return value;
}
}
}

113
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java Normal file
View File

@@ -0,0 +1,113 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.util.*;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
/**
* Message对象解析工具类
*
* @author machengyuan 2018-8-19 下午06:14:23
* @version 1.0.0
*/
public class MessageUtil {
public static void parse4Dml(Message message, Consumer<Dml> consumer) {
if (message == null) {
return;
}
List<CanalEntry.Entry> entries = message.getEntries();
for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e);
}
CanalEntry.EventType eventType = rowChange.getEventType();
final Dml dml = new Dml();
dml.setDatabase(entry.getHeader().getSchemaName());
dml.setTable(entry.getHeader().getTableName());
dml.setType(eventType.toString());
dml.setTs(System.currentTimeMillis());
dml.setSql(rowChange.getSql());
List<Map<String, Object>> data = new ArrayList<>();
List<Map<String, Object>> old = new ArrayList<>();
if (!rowChange.getIsDdl()) {
Set<String> updateSet = new HashSet<>();
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
List<CanalEntry.Column> columns;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
for (CanalEntry.Column column : columns) {
row.put(column.getName(),
JdbcTypeUtil.typeConvert(dml.getTable(),
column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
// 获取update为true的字段
if (column.getUpdated()) {
updateSet.add(column.getName());
}
}
if (!row.isEmpty()) {
data.add(row);
}
if (eventType == CanalEntry.EventType.UPDATE) {
Map<String, Object> rowOld = new LinkedHashMap<>();
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
rowOld.put(column.getName(),
JdbcTypeUtil.typeConvert(dml.getTable(),
column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
}
}
// update操作将记录修改前的值
if (!rowOld.isEmpty()) {
old.add(rowOld);
}
}
}
if (!data.isEmpty()) {
dml.setData(data);
}
if (!old.isEmpty()) {
dml.setOld(old);
}
consumer.accept(dml);
}
}
}
public interface Consumer<T> {
void accept(T t);
}
}

12
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java Normal file
View File

@@ -0,0 +1,12 @@
package com.alibaba.otter.canal.client.adapter.support;
import java.lang.annotation.*;
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE })
public @interface SPI {
// Default SPI name
String value() default "";
}