chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

整理代码

Browse Source
This commit is contained in:
rewer
2019-02-07 13:49:35 +08:00
parent ad62275cd7
commit 0c5f5c128f
14 changed files with 3253 additions and 3243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

312
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java
View File

@@ -1,156 +1,156 @@
package com.alibaba.otter.canal.client.adapter.config; package com.alibaba.otter.canal.client.adapter.config;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import org.springframework.util.PropertyPlaceholderHelper; import org.springframework.util.PropertyPlaceholderHelper;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory;
import com.alibaba.otter.canal.client.adapter.config.common.*; import com.alibaba.otter.canal.client.adapter.config.common.*;
/** /**
* 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范
* *
* @author reweerma 2019-2-1 上午9:14:02 * @author reweerma 2019-2-1 上午9:14:02
* @version 1.0.0 * @version 1.0.0
*/ */
public class YmlConfigBinder { public class YmlConfigBinder {
/** /**
* 将当前内容绑定到指定对象 * 将当前内容绑定到指定对象
* *
* @param content yml内容 * @param content yml内容
* @param clazz 指定对象类型 * @param clazz 指定对象类型
* @return 对象 * @return 对象
*/ */
public static <T> T bindYmlToObj(String content, Class<T> clazz) { public static <T> T bindYmlToObj(String content, Class<T> clazz) {
return bindYmlToObj(null, content, clazz, null); return bindYmlToObj(null, content, clazz, null);
} }
/** /**
* 将当前内容绑定到指定对象并指定内容编码格式 * 将当前内容绑定到指定对象并指定内容编码格式
* *
* @param content yml内容 * @param content yml内容
* @param clazz 指定对象类型 * @param clazz 指定对象类型
* @param charset yml内容编码格式 * @param charset yml内容编码格式
* @return 对象 * @return 对象
*/ */
public static <T> T bindYmlToObj(String content, Class<T> clazz, String charset) { public static <T> T bindYmlToObj(String content, Class<T> clazz, String charset) {
return bindYmlToObj(null, content, clazz, charset); return bindYmlToObj(null, content, clazz, charset);
} }
/** /**
* 将当前内容指定前缀部分绑定到指定对象 * 将当前内容指定前缀部分绑定到指定对象
* *
* @param prefix 指定前缀 * @param prefix 指定前缀
* @param content yml内容 * @param content yml内容
* @param clazz 指定对象类型 * @param clazz 指定对象类型
* @return 对象 * @return 对象
*/ */
public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz) { public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz) {
return bindYmlToObj(prefix, content, clazz, null); return bindYmlToObj(prefix, content, clazz, null);
} }
/** /**
* 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式
* *
* @param prefix 指定前缀 * @param prefix 指定前缀
* @param content yml内容 * @param content yml内容
* @param clazz 指定对象类型 * @param clazz 指定对象类型
* @param charset yml内容编码格式 * @param charset yml内容编码格式
* @return 对象 * @return 对象
*/ */
public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset) { public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset) {
return bindYmlToObj(prefix, content, clazz, charset, null); return bindYmlToObj(prefix, content, clazz, charset, null);
} }
/** /**
* 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers} * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers:
* 在envProperties中有属性 zookeeper.servers: * ${zookeeper.servers} 在envProperties中有属性 zookeeper.servers:
* 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为 * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为
* zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中 * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中
* zkServers 属性是 List<String> 对象, 则会自动映射成List * zkServers 属性是 List<String> 对象, 则会自动映射成List
* *
* @param prefix 指定前缀 * @param prefix 指定前缀
* @param content yml内容 * @param content yml内容
* @param clazz 指定对象类型 * @param clazz 指定对象类型
* @param charset yml内容编码格式 * @param charset yml内容编码格式
* @return 对象 * @return 对象
*/ */
public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset, public static <T> T bindYmlToObj(String prefix, String content, Class<T> clazz, String charset,
Properties baseProperties) { Properties baseProperties) {
try { try {
byte[] contentBytes; byte[] contentBytes;
if (charset == null) { if (charset == null) {
contentBytes = content.getBytes(); contentBytes = content.getBytes();
} else { } else {
contentBytes = content.getBytes(charset); contentBytes = content.getBytes(charset);
} }
YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
Resource configResource = new ByteArrayResource(contentBytes); Resource configResource = new ByteArrayResource(contentBytes);
PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); PropertySource<?> propertySource = propertySourceLoader.load("manualBindConfig", configResource, null);
if (propertySource == null) { if (propertySource == null) {
return null; return null;
} }
Properties properties = new Properties(); Properties properties = new Properties();
Map<String, Object> propertiesRes = new LinkedHashMap<>(); Map<String, Object> propertiesRes = new LinkedHashMap<>();
if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) {
prefix = prefix + "."; prefix = prefix + ".";
} }
properties.putAll((Map) propertySource.getSource()); properties.putAll((Map<?, ?>) propertySource.getSource());
if (baseProperties != null) { if (baseProperties != null) {
baseProperties.putAll(properties); baseProperties.putAll(properties);
properties = baseProperties; properties = baseProperties;
} }
for (Object o : ((Map) propertySource.getSource()).entrySet()) { for (Object o : ((Map<?, ?>) propertySource.getSource()).entrySet()) {
Map.Entry entry = (Map.Entry) o; Map.Entry<?, ?> entry = (Map.Entry<?, ?>) o;
String key = (String) entry.getKey(); String key = (String) entry.getKey();
Object value = entry.getValue(); Object value = entry.getValue();
if (prefix != null) { if (prefix != null) {
if (key != null && key.startsWith(prefix)) { if (key != null && key.startsWith(prefix)) {
key = key.substring(prefix.length()); key = key.substring(prefix.length());
} else { } else {
continue; continue;
} }
} }
if (value != null && value.toString().contains("${")) { if (value != null && value.toString().contains("${")) {
PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}");
value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties); value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties);
} }
propertiesRes.put(key, value); propertiesRes.put(key, value);
} }
if (propertiesRes.isEmpty()) { if (propertiesRes.isEmpty()) {
return null; return null;
} }
propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); propertySource = new MapPropertySource(propertySource.getName(), propertiesRes);
T target = clazz.newInstance(); T target = clazz.newInstance();
MutablePropertySources propertySources = new MutablePropertySources(); MutablePropertySources propertySources = new MutablePropertySources();
propertySources.addFirst(propertySource); propertySources.addFirst(propertySource);
PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target); PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target);
factory.setPropertySources(propertySources); factory.setPropertySources(propertySources);
factory.setIgnoreInvalidFields(true); factory.setIgnoreInvalidFields(true);
factory.setIgnoreUnknownFields(true); factory.setIgnoreUnknownFields(true);
factory.bindPropertiesToTarget(); factory.bindPropertiesToTarget();
return target; return target;
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
} }
} }

104
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java
View File

@@ -1,52 +1,52 @@
package com.alibaba.otter.canal.client.adapter.config.bind; package com.alibaba.otter.canal.client.adapter.config.bind;
import org.springframework.beans.PropertyValue; import org.springframework.beans.PropertyValue;
import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; import com.alibaba.otter.canal.client.adapter.config.common.PropertySource;
/** /**
* A {@link PropertyValue} that can provide information about its origin. * A {@link PropertyValue} that can provide information about its origin.
* *
* @author Andy Wilkinson * @author Andy Wilkinson
*/ */
class OriginCapablePropertyValue extends PropertyValue { class OriginCapablePropertyValue extends PropertyValue {
private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin";
private final PropertyOrigin origin; private final PropertyOrigin origin;
OriginCapablePropertyValue(PropertyValue propertyValue){ OriginCapablePropertyValue(PropertyValue propertyValue){
this(propertyValue.getName(), this(propertyValue.getName(),
propertyValue.getValue(), propertyValue.getValue(),
(PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN));
} }
OriginCapablePropertyValue(String name, Object value, String originName, PropertySource<?> originSource){ OriginCapablePropertyValue(String name, Object value, String originName, PropertySource<?> originSource){
this(name, value, new PropertyOrigin(originSource, originName)); this(name, value, new PropertyOrigin(originSource, originName));
} }
OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){
super(name, value); super(name, value);
this.origin = origin; this.origin = origin;
setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin);
} }
public PropertyOrigin getOrigin() { public PropertyOrigin getOrigin() {
return this.origin; return this.origin;
} }
@Override @Override
public String toString() { public String toString() {
String name = (this.origin != null ? this.origin.getName() : this.getName()); String name = (this.origin != null ? this.origin.getName() : this.getName());
String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown");
return "'" + name + "' from '" + source + "'"; return "'" + name + "' from '" + source + "'";
} }
public static PropertyOrigin getOrigin(PropertyValue propertyValue) { public static PropertyOrigin getOrigin(PropertyValue propertyValue) {
if (propertyValue instanceof OriginCapablePropertyValue) { if (propertyValue instanceof OriginCapablePropertyValue) {
return ((OriginCapablePropertyValue) propertyValue).getOrigin(); return ((OriginCapablePropertyValue) propertyValue).getOrigin();
} }
return new OriginCapablePropertyValue(propertyValue).getOrigin(); return new OriginCapablePropertyValue(propertyValue).getOrigin();
} }
} }

235
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java
View File

@@ -1,118 +1,117 @@
package com.alibaba.otter.canal.client.adapter.config.common; package com.alibaba.otter.canal.client.adapter.config.common;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Arrays; import java.util.Arrays;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.InputStreamSource; import org.springframework.core.io.Resource;
import org.springframework.core.io.Resource; import org.springframework.util.Assert;
import org.springframework.util.Assert;
/**
/** * {@link Resource} implementation for a given byte array.
* {@link Resource} implementation for a given byte array. * <p>
* <p> * Creates a {@link ByteArrayInputStream} for the given byte array.
* Creates a {@link ByteArrayInputStream} for the given byte array. * <p>
* <p> * Useful for loading content from any given byte array, without having to
* Useful for loading content from any given byte array, without having to * resort to a single-use {@link InputStreamResource}. Particularly useful for
* resort to a single-use {@link InputStreamResource}. Particularly useful for * creating mail attachments from local content, where JavaMail needs to be able
* creating mail attachments from local content, where JavaMail needs to be able * to read the stream multiple times.
* to read the stream multiple times. *
* * @author Juergen Hoeller
* @author Juergen Hoeller * @author Sam Brannen
* @author Sam Brannen * @see ByteArrayInputStream
* @see ByteArrayInputStream * @see InputStreamResource
* @see InputStreamResource * @since 1.2.3
* @since 1.2.3 */
*/ public class ByteArrayResource extends AbstractResource {
public class ByteArrayResource extends AbstractResource {
private final byte[] byteArray;
private final byte[] byteArray;
private final String description;
private final String description;
/**
/** * Create a new {@code ByteArrayResource}.
* Create a new {@code ByteArrayResource}. *
* * @param byteArray the byte array to wrap
* @param byteArray the byte array to wrap */
*/ public ByteArrayResource(byte[] byteArray){
public ByteArrayResource(byte[] byteArray){ this(byteArray, "resource loaded from byte array");
this(byteArray, "resource loaded from byte array"); }
}
/**
/** * Create a new {@code ByteArrayResource} with a description.
* Create a new {@code ByteArrayResource} with a description. *
* * @param byteArray the byte array to wrap
* @param byteArray the byte array to wrap * @param description where the byte array comes from
* @param description where the byte array comes from */
*/ public ByteArrayResource(byte[] byteArray, String description){
public ByteArrayResource(byte[] byteArray, String description){ Assert.notNull(byteArray, "Byte array must not be null");
Assert.notNull(byteArray, "Byte array must not be null"); this.byteArray = byteArray;
this.byteArray = byteArray; this.description = (description != null ? description : "");
this.description = (description != null ? description : ""); }
}
/**
/** * Return the underlying byte array.
* Return the underlying byte array. */
*/ public final byte[] getByteArray() {
public final byte[] getByteArray() { return this.byteArray;
return this.byteArray; }
}
/**
/** * This implementation always returns {@code true}.
* This implementation always returns {@code true}. */
*/ @Override
@Override public boolean exists() {
public boolean exists() { return true;
return true; }
}
/**
/** * This implementation returns the length of the underlying byte array.
* This implementation returns the length of the underlying byte array. */
*/ @Override
@Override public long contentLength() {
public long contentLength() { return this.byteArray.length;
return this.byteArray.length; }
}
/**
/** * This implementation returns a ByteArrayInputStream for the underlying byte
* This implementation returns a ByteArrayInputStream for the underlying byte * array.
* array. *
* * @see ByteArrayInputStream
* @see ByteArrayInputStream */
*/ @Override
@Override public InputStream getInputStream() throws IOException {
public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(this.byteArray);
return new ByteArrayInputStream(this.byteArray); }
}
/**
/** * This implementation returns a description that includes the passed-in
* This implementation returns a description that includes the passed-in * {@code description}, if any.
* {@code description}, if any. */
*/ @Override
@Override public String getDescription() {
public String getDescription() { return "Byte array resource [" + this.description + "]";
return "Byte array resource [" + this.description + "]"; }
}
/**
/** * This implementation compares the underlying byte array.
* This implementation compares the underlying byte array. *
* * @see Arrays#equals(byte[], byte[])
* @see Arrays#equals(byte[], byte[]) */
*/ @Override
@Override public boolean equals(Object obj) {
public boolean equals(Object obj) { return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource
return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray)));
&& Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); }
}
/**
/** * This implementation returns the hash code based on the underlying byte array.
* This implementation returns the hash code based on the underlying byte array. */
*/ @Override
@Override public int hashCode() {
public int hashCode() { return (byte[].class.hashCode() * 29 * this.byteArray.length);
return (byte[].class.hashCode() * 29 * this.byteArray.length); }
}
}
}

844
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java
View File

@@ -1,422 +1,422 @@
package com.alibaba.otter.canal.client.adapter.es.config; package com.alibaba.otter.canal.client.adapter.es.config;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
/** /**
* ES 映射配置视图 * ES 映射配置视图
* *
* @author rewerma 2018-11-01 * @author rewerma 2018-11-01
* @version 1.0.0 * @version 1.0.0
*/ */
public class SchemaItem { public class SchemaItem {
private Map<String, TableItem> aliasTableItems = new LinkedHashMap<>(); // 别名对应表名 private Map<String, TableItem> aliasTableItems = new LinkedHashMap<>(); // 别名对应表名
private Map<String, FieldItem> selectFields = new LinkedHashMap<>(); // 查询字段 private Map<String, FieldItem> selectFields = new LinkedHashMap<>(); // 查询字段
private String sql; private String sql;
private volatile Map<String, List<TableItem>> tableItemAliases; private volatile Map<String, List<TableItem>> tableItemAliases;
private volatile Map<String, List<FieldItem>> columnFields; private volatile Map<String, List<FieldItem>> columnFields;
private volatile Boolean allFieldsSimple; private volatile Boolean allFieldsSimple;
public void init() { public void init() {
this.getTableItemAliases(); this.getTableItemAliases();
this.getColumnFields(); this.getColumnFields();
this.isAllFieldsSimple(); this.isAllFieldsSimple();
aliasTableItems.values().forEach(tableItem -> { aliasTableItems.values().forEach(tableItem -> {
tableItem.getRelationTableFields(); tableItem.getRelationTableFields();
tableItem.getRelationSelectFieldItems(); tableItem.getRelationSelectFieldItems();
}); });
} }
public Map<String, TableItem> getAliasTableItems() { public Map<String, TableItem> getAliasTableItems() {
return aliasTableItems; return aliasTableItems;
} }
public void setAliasTableItems(Map<String, TableItem> aliasTableItems) { public void setAliasTableItems(Map<String, TableItem> aliasTableItems) {
this.aliasTableItems = aliasTableItems; this.aliasTableItems = aliasTableItems;
} }
public String getSql() { public String getSql() {
return sql; return sql;
} }
public void setSql(String sql) { public void setSql(String sql) {
this.sql = sql; this.sql = sql;
} }
public Map<String, FieldItem> getSelectFields() { public Map<String, FieldItem> getSelectFields() {
return selectFields; return selectFields;
} }
public void setSelectFields(Map<String, FieldItem> selectFields) { public void setSelectFields(Map<String, FieldItem> selectFields) {
this.selectFields = selectFields; this.selectFields = selectFields;
} }
public Map<String, List<TableItem>> getTableItemAliases() { public Map<String, List<TableItem>> getTableItemAliases() {
if (tableItemAliases == null) { if (tableItemAliases == null) {
synchronized (SchemaItem.class) { synchronized (SchemaItem.class) {
if (tableItemAliases == null) { if (tableItemAliases == null) {
tableItemAliases = new LinkedHashMap<>(); tableItemAliases = new LinkedHashMap<>();
aliasTableItems.forEach((alias, tableItem) -> { aliasTableItems.forEach((alias, tableItem) -> {
List<TableItem> aliases = tableItemAliases List<TableItem> aliases = tableItemAliases
.computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>()); .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>());
aliases.add(tableItem); aliases.add(tableItem);
}); });
} }
} }
} }
return tableItemAliases; return tableItemAliases;
} }
public Map<String, List<FieldItem>> getColumnFields() { public Map<String, List<FieldItem>> getColumnFields() {
if (columnFields == null) { if (columnFields == null) {
synchronized (SchemaItem.class) { synchronized (SchemaItem.class) {
if (columnFields == null) { if (columnFields == null) {
columnFields = new LinkedHashMap<>(); columnFields = new LinkedHashMap<>();
getSelectFields() getSelectFields()
.forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> {
TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); // TableItem tableItem = getAliasTableItems().get(columnItem.getOwner());
// if (!tableItem.isSubQuery()) { // if (!tableItem.isSubQuery()) {
List<FieldItem> fieldItems = columnFields.computeIfAbsent( List<FieldItem> fieldItems = columnFields.computeIfAbsent(
columnItem.getOwner() + "." + columnItem.getColumnName(), columnItem.getOwner() + "." + columnItem.getColumnName(),
k -> new ArrayList<>()); k -> new ArrayList<>());
fieldItems.add(fieldItem); fieldItems.add(fieldItem);
// } else { // } else {
// tableItem.getSubQueryFields().forEach(subQueryField -> { // tableItem.getSubQueryFields().forEach(subQueryField -> {
// List<FieldItem> fieldItems = columnFields.computeIfAbsent( // List<FieldItem> fieldItems = columnFields.computeIfAbsent(
// columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(),
// k -> new ArrayList<>()); // k -> new ArrayList<>());
// fieldItems.add(fieldItem); // fieldItems.add(fieldItem);
// }); // });
// } // }
})); }));
} }
} }
} }
return columnFields; return columnFields;
} }
public boolean isAllFieldsSimple() { public boolean isAllFieldsSimple() {
if (allFieldsSimple == null) { if (allFieldsSimple == null) {
synchronized (SchemaItem.class) { synchronized (SchemaItem.class) {
if (allFieldsSimple == null) { if (allFieldsSimple == null) {
allFieldsSimple = true; allFieldsSimple = true;
for (FieldItem fieldItem : getSelectFields().values()) { for (FieldItem fieldItem : getSelectFields().values()) {
if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
allFieldsSimple = false; allFieldsSimple = false;
break; break;
} }
} }
} }
} }
} }
return allFieldsSimple; return allFieldsSimple;
} }
public TableItem getMainTable() { public TableItem getMainTable() {
if (!aliasTableItems.isEmpty()) { if (!aliasTableItems.isEmpty()) {
return aliasTableItems.values().iterator().next(); return aliasTableItems.values().iterator().next();
} else { } else {
return null; return null;
} }
} }
public FieldItem getIdFieldItem(ESMapping mapping) { public FieldItem getIdFieldItem(ESMapping mapping) {
if (mapping.get_id() != null) { if (mapping.get_id() != null) {
return getSelectFields().get(mapping.get_id()); return getSelectFields().get(mapping.get_id());
} else { } else {
return getSelectFields().get(mapping.getPk()); return getSelectFields().get(mapping.getPk());
} }
} }
public static class TableItem { public static class TableItem {
private SchemaItem schemaItem; private SchemaItem schemaItem;
private String schema; private String schema;
private String tableName; private String tableName;
private String alias; private String alias;
private String subQuerySql; private String subQuerySql;
private List<FieldItem> subQueryFields = new ArrayList<>(); private List<FieldItem> subQueryFields = new ArrayList<>();
private List<RelationFieldsPair> relationFields = new ArrayList<>(); private List<RelationFieldsPair> relationFields = new ArrayList<>();
private boolean main; private boolean main;
private boolean subQuery; private boolean subQuery;
private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段对应主表查询字段 private volatile Map<FieldItem, List<FieldItem>> relationTableFields; // 当前表关联条件字段对应主表查询字段
private volatile List<FieldItem> relationSelectFieldItems; // 子表所在主表的查询字段 private volatile List<FieldItem> relationSelectFieldItems; // 子表所在主表的查询字段
public TableItem(SchemaItem schemaItem){ public TableItem(SchemaItem schemaItem){
this.schemaItem = schemaItem; this.schemaItem = schemaItem;
} }
public SchemaItem getSchemaItem() { public SchemaItem getSchemaItem() {
return schemaItem; return schemaItem;
} }
public void setSchemaItem(SchemaItem schemaItem) { public void setSchemaItem(SchemaItem schemaItem) {
this.schemaItem = schemaItem; this.schemaItem = schemaItem;
} }
public String getSchema() { public String getSchema() {
return schema; return schema;
} }
public void setSchema(String schema) { public void setSchema(String schema) {
this.schema = schema; this.schema = schema;
} }
public String getTableName() { public String getTableName() {
return tableName; return tableName;
} }
public void setTableName(String tableName) { public void setTableName(String tableName) {
this.tableName = tableName; this.tableName = tableName;
} }
public String getAlias() { public String getAlias() {
return alias; return alias;
} }
public void setAlias(String alias) { public void setAlias(String alias) {
this.alias = alias; this.alias = alias;
} }
public String getSubQuerySql() { public String getSubQuerySql() {
return subQuerySql; return subQuerySql;
} }
public void setSubQuerySql(String subQuerySql) { public void setSubQuerySql(String subQuerySql) {
this.subQuerySql = subQuerySql; this.subQuerySql = subQuerySql;
} }
public boolean isMain() { public boolean isMain() {
return main; return main;
} }
public void setMain(boolean main) { public void setMain(boolean main) {
this.main = main; this.main = main;
} }
public boolean isSubQuery() { public boolean isSubQuery() {
return subQuery; return subQuery;
} }
public void setSubQuery(boolean subQuery) { public void setSubQuery(boolean subQuery) {
this.subQuery = subQuery; this.subQuery = subQuery;
} }
public List<FieldItem> getSubQueryFields() { public List<FieldItem> getSubQueryFields() {
return subQueryFields; return subQueryFields;
} }
public void setSubQueryFields(List<FieldItem> subQueryFields) { public void setSubQueryFields(List<FieldItem> subQueryFields) {
this.subQueryFields = subQueryFields; this.subQueryFields = subQueryFields;
} }
public List<RelationFieldsPair> getRelationFields() { public List<RelationFieldsPair> getRelationFields() {
return relationFields; return relationFields;
} }
public void setRelationFields(List<RelationFieldsPair> relationFields) { public void setRelationFields(List<RelationFieldsPair> relationFields) {
this.relationFields = relationFields; this.relationFields = relationFields;
} }
public Map<FieldItem, List<FieldItem>> getRelationTableFields() { public Map<FieldItem, List<FieldItem>> getRelationTableFields() {
if (relationTableFields == null) { if (relationTableFields == null) {
synchronized (SchemaItem.class) { synchronized (SchemaItem.class) {
if (relationTableFields == null) { if (relationTableFields == null) {
relationTableFields = new LinkedHashMap<>(); relationTableFields = new LinkedHashMap<>();
getRelationFields().forEach(relationFieldsPair -> { getRelationFields().forEach(relationFieldsPair -> {
FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
FieldItem currentTableRelField = null; FieldItem currentTableRelField = null;
if (getAlias().equals(leftFieldItem.getOwner())) { if (getAlias().equals(leftFieldItem.getOwner())) {
currentTableRelField = leftFieldItem; currentTableRelField = leftFieldItem;
} else if (getAlias().equals(rightFieldItem.getOwner())) { } else if (getAlias().equals(rightFieldItem.getOwner())) {
currentTableRelField = rightFieldItem; currentTableRelField = rightFieldItem;
} }
if (currentTableRelField != null) { if (currentTableRelField != null) {
List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields() List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
.get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName());
if (selectFieldItem != null && !selectFieldItem.isEmpty()) { if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
relationTableFields.put(currentTableRelField, selectFieldItem); relationTableFields.put(currentTableRelField, selectFieldItem);
} else { } else {
selectFieldItem = getSchemaItem().getColumnFields() selectFieldItem = getSchemaItem().getColumnFields()
.get(rightFieldItem.getOwner() + "." .get(rightFieldItem.getOwner() + "."
+ rightFieldItem.getColumn().getColumnName()); + rightFieldItem.getColumn().getColumnName());
if (selectFieldItem != null && !selectFieldItem.isEmpty()) { if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
relationTableFields.put(currentTableRelField, selectFieldItem); relationTableFields.put(currentTableRelField, selectFieldItem);
} else { } else {
throw new UnsupportedOperationException( throw new UnsupportedOperationException(
"Relation condition column must in select columns."); "Relation condition column must in select columns.");
} }
} }
} }
}); });
} }
} }
} }
return relationTableFields; return relationTableFields;
} }
public List<FieldItem> getRelationSelectFieldItems() { public List<FieldItem> getRelationSelectFieldItems() {
if (relationSelectFieldItems == null) { if (relationSelectFieldItems == null) {
synchronized (SchemaItem.class) { synchronized (SchemaItem.class) {
if (relationSelectFieldItems == null) { if (relationSelectFieldItems == null) {
relationSelectFieldItems = new ArrayList<>(); relationSelectFieldItems = new ArrayList<>();
for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
if (fieldItem.getOwners().contains(getAlias())) { if (fieldItem.getOwners().contains(getAlias())) {
relationSelectFieldItems.add(fieldItem); relationSelectFieldItems.add(fieldItem);
} }
} }
} }
} }
} }
return relationSelectFieldItems; return relationSelectFieldItems;
} }
} }
public static class RelationFieldsPair { public static class RelationFieldsPair {
private FieldItem leftFieldItem; private FieldItem leftFieldItem;
private FieldItem rightFieldItem; private FieldItem rightFieldItem;
public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){ public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){
this.leftFieldItem = leftFieldItem; this.leftFieldItem = leftFieldItem;
this.rightFieldItem = rightFieldItem; this.rightFieldItem = rightFieldItem;
} }
public FieldItem getLeftFieldItem() { public FieldItem getLeftFieldItem() {
return leftFieldItem; return leftFieldItem;
} }
public void setLeftFieldItem(FieldItem leftFieldItem) { public void setLeftFieldItem(FieldItem leftFieldItem) {
this.leftFieldItem = leftFieldItem; this.leftFieldItem = leftFieldItem;
} }
public FieldItem getRightFieldItem() { public FieldItem getRightFieldItem() {
return rightFieldItem; return rightFieldItem;
} }
public void setRightFieldItem(FieldItem rightFieldItem) { public void setRightFieldItem(FieldItem rightFieldItem) {
this.rightFieldItem = rightFieldItem; this.rightFieldItem = rightFieldItem;
} }
} }
public static class FieldItem { public static class FieldItem {
private String fieldName; private String fieldName;
private List<ColumnItem> columnItems = new ArrayList<>(); private List<ColumnItem> columnItems = new ArrayList<>();
private List<String> owners = new ArrayList<>(); private List<String> owners = new ArrayList<>();
private boolean method; private boolean method;
private boolean binaryOp; private boolean binaryOp;
public String getFieldName() { public String getFieldName() {
return fieldName; return fieldName;
} }
public void setFieldName(String fieldName) { public void setFieldName(String fieldName) {
this.fieldName = fieldName; this.fieldName = fieldName;
} }
public List<ColumnItem> getColumnItems() { public List<ColumnItem> getColumnItems() {
return columnItems; return columnItems;
} }
public void setColumnItems(List<ColumnItem> columnItems) { public void setColumnItems(List<ColumnItem> columnItems) {
this.columnItems = columnItems; this.columnItems = columnItems;
} }
public boolean isMethod() { public boolean isMethod() {
return method; return method;
} }
public void setMethod(boolean method) { public void setMethod(boolean method) {
this.method = method; this.method = method;
} }
public boolean isBinaryOp() { public boolean isBinaryOp() {
return binaryOp; return binaryOp;
} }
public void setBinaryOp(boolean binaryOp) { public void setBinaryOp(boolean binaryOp) {
this.binaryOp = binaryOp; this.binaryOp = binaryOp;
} }
public List<String> getOwners() { public List<String> getOwners() {
return owners; return owners;
} }
public void setOwners(List<String> owners) { public void setOwners(List<String> owners) {
this.owners = owners; this.owners = owners;
} }
public void addColumn(ColumnItem columnItem) { public void addColumn(ColumnItem columnItem) {
columnItems.add(columnItem); columnItems.add(columnItem);
} }
public ColumnItem getColumn() { public ColumnItem getColumn() {
if (!columnItems.isEmpty()) { if (!columnItems.isEmpty()) {
return columnItems.get(0); return columnItems.get(0);
} else { } else {
return null; return null;
} }
} }
public String getOwner() { public String getOwner() {
if (!owners.isEmpty()) { if (!owners.isEmpty()) {
return owners.get(0); return owners.get(0);
} else { } else {
return null; return null;
} }
} }
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
FieldItem fieldItem = (FieldItem) o; FieldItem fieldItem = (FieldItem) o;
return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null; return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null;
} }
@Override @Override
public int hashCode() { public int hashCode() {
return fieldName != null ? fieldName.hashCode() : 0; return fieldName != null ? fieldName.hashCode() : 0;
} }
} }
public static class ColumnItem { public static class ColumnItem {
private String owner; private String owner;
private String columnName; private String columnName;
public String getOwner() { public String getOwner() {
return owner; return owner;
} }
public void setOwner(String owner) { public void setOwner(String owner) {
this.owner = owner; this.owner = owner;
} }
public String getColumnName() { public String getColumnName() {
return columnName; return columnName;
} }
public void setColumnName(String columnName) { public void setColumnName(String columnName) {
this.columnName = columnName; this.columnName = columnName;
} }
} }
} }

1052
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java
View File

File diff suppressed because it is too large Load Diff

789
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -1,392 +1,397 @@
package com.alibaba.otter.canal.client.adapter.hbase.service; package com.alibaba.otter.canal.client.adapter.hbase.service;
import java.sql.*; import java.sql.ResultSetMetaData;
import java.text.SimpleDateFormat; import java.sql.Timestamp;
import java.util.ArrayList; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.ArrayList;
import java.util.List; import java.util.Date;
import java.util.concurrent.ExecutorService; import java.util.List;
import java.util.concurrent.Executors; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function; import java.util.concurrent.atomic.AtomicLong;
import javax.sql.DataSource; import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.support.Util; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger;
import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
import com.alibaba.otter.canal.client.adapter.hbase.support.*; import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
import com.google.common.base.Joiner; import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
/** import com.alibaba.otter.canal.client.adapter.support.EtlResult;
* HBase ETL 操作业务类 import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
* import com.alibaba.otter.canal.client.adapter.support.Util;
* @author rewerma @ 2018-10-20 import com.google.common.base.Joiner;
* @version 1.0.0
*/ /**
public class HbaseEtlService { * HBase ETL 操作业务类
*
private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); * @author rewerma @ 2018-10-20
* @version 1.0.0
*/
/** public class HbaseEtlService {
* 建表
* private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class);
* @param hbaseTemplate
* @param config
*/ /**
public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { * 建表
try { *
// 判断hbase表是否存在，不存在则建表 * @param hbaseTemplate
MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); * @param config
if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { */
hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily()); public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) {
} try {
} catch (Exception e) { // 判断hbase表是否存在，不存在则建表
logger.error(e.getMessage(), e); MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
throw new RuntimeException(e); if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
} hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily());
} }
} catch (Exception e) {
/** logger.error(e.getMessage(), e);
* 导入数据 throw new RuntimeException(e);
* }
* @param ds 数据源 }
* @param hbaseTemplate hbaseTemplate
* @param config 配置 /**
* @param params 筛选条件 * 导入数据
* @return 导入结果 *
*/ * @param ds 数据源
public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, * @param hbaseTemplate hbaseTemplate
List<String> params) { * @param config 配置
EtlResult etlResult = new EtlResult(); * @param params 筛选条件
AtomicLong successCount = new AtomicLong(); * @return 导入结果
List<String> errMsg = new ArrayList<>(); */
String hbaseTable = ""; public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config,
try { List<String> params) {
if (config == null) { EtlResult etlResult = new EtlResult();
logger.error("Config is null!"); AtomicLong successCount = new AtomicLong();
etlResult.setSucceeded(false); List<String> errMsg = new ArrayList<>();
etlResult.setErrorMessage("Config is null!"); String hbaseTable = "";
return etlResult; try {
} if (config == null) {
MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); logger.error("Config is null!");
hbaseTable = hbaseMapping.getHbaseTable(); etlResult.setSucceeded(false);
etlResult.setErrorMessage("Config is null!");
long start = System.currentTimeMillis(); return etlResult;
}
if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!"); hbaseTable = hbaseMapping.getHbaseTable();
// 如果表存在则删除
if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { long start = System.currentTimeMillis();
hbaseTemplate.disableTable(hbaseMapping.getHbaseTable());
hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable()); if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) {
} logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!");
params = null; // 如果表存在则删除
} else { if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
logger.info(hbaseMapping.getHbaseTable() + " etl is starting!"); hbaseTemplate.disableTable(hbaseMapping.getHbaseTable());
} hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable());
createTable(hbaseTemplate, config); }
params = null;
// 拼接sql } else {
String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable(); logger.info(hbaseMapping.getHbaseTable() + " etl is starting!");
}
// 拼接条件 createTable(hbaseTemplate, config);
if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) {
AtomicBoolean stExists = new AtomicBoolean(false); // 拼接sql
// 验证是否有SYS_TIME字段 String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable();
Util.sqlRS(ds, sql, rs -> {
try { // 拼接条件
ResultSetMetaData rsmd = rs.getMetaData(); if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) {
int cnt = rsmd.getColumnCount(); AtomicBoolean stExists = new AtomicBoolean(false);
for (int i = 1; i <= cnt; i++) { // 验证是否有SYS_TIME字段
String columnName = rsmd.getColumnName(i); Util.sqlRS(ds, sql, rs -> {
if ("SYS_TIME".equalsIgnoreCase(columnName)) { try {
stExists.set(true); ResultSetMetaData rsmd = rs.getMetaData();
break; int cnt = rsmd.getColumnCount();
} for (int i = 1; i <= cnt; i++) {
} String columnName = rsmd.getColumnName(i);
} catch (Exception e) { if ("SYS_TIME".equalsIgnoreCase(columnName)) {
// ignore stExists.set(true);
} break;
return null; }
}); }
if (stExists.get()) { } catch (Exception e) {
sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; // ignore
} }
} else if (hbaseMapping.getEtlCondition() != null && params != null) { return null;
String etlCondition = hbaseMapping.getEtlCondition(); });
int size = params.size(); if (stExists.get()) {
for (int i = 0; i < size; i++) { sql += " WHERE SYS_TIME >= '" + params.get(0) + "' ";
etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); }
} } else if (hbaseMapping.getEtlCondition() != null && params != null) {
String etlCondition = hbaseMapping.getEtlCondition();
sql += " " + etlCondition; int size = params.size();
} for (int i = 0; i < size; i++) {
etlCondition = etlCondition.replace("{" + i + "}", params.get(i));
// 获取总数 }
String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
long cnt = (Long) Util.sqlRS(ds, countSql, rs -> { sql += " " + etlCondition;
Long count = null; }
try {
if (rs.next()) { // 获取总数
count = ((Number) rs.getObject(1)).longValue(); String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
} long cnt = (Long) Util.sqlRS(ds, countSql, rs -> {
} catch (Exception e) { Long count = null;
logger.error(e.getMessage(), e); try {
} if (rs.next()) {
return count == null ? 0 : count; count = ((Number) rs.getObject(1)).longValue();
}); }
} catch (Exception e) {
// 当大于1万条记录时开启多线程 logger.error(e.getMessage(), e);
if (cnt >= 10000) { }
int threadCount = 3; return count == null ? 0 : count;
long perThreadCnt = cnt / threadCount; });
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
List<Future<Boolean>> futures = new ArrayList<>(threadCount); // 当大于1万条记录时开启多线程
for (int i = 0; i < threadCount; i++) { if (cnt >= 10000) {
long offset = i * perThreadCnt; int threadCount = 3;
Long size = null; long perThreadCnt = cnt / threadCount;
if (i != threadCount - 1) { ExecutorService executor = Executors.newFixedThreadPool(threadCount);
size = perThreadCnt; List<Future<Boolean>> futures = new ArrayList<>(threadCount);
} for (int i = 0; i < threadCount; i++) {
String sqlFinal; long offset = i * perThreadCnt;
if (size != null) { Long size = null;
sqlFinal = sql + " LIMIT " + offset + "," + size; if (i != threadCount - 1) {
} else { size = perThreadCnt;
sqlFinal = sql + " LIMIT " + offset + "," + cnt; }
} String sqlFinal;
Future<Boolean> future = executor.submit( if (size != null) {
() -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg)); sqlFinal = sql + " LIMIT " + offset + "," + size;
futures.add(future); } else {
} sqlFinal = sql + " LIMIT " + offset + "," + cnt;
}
for (Future<Boolean> future : futures) { Future<Boolean> future = executor.submit(
future.get(); () -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg));
} futures.add(future);
}
executor.shutdown();
} else { for (Future<Boolean> future : futures) {
executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg); future.get();
} }
logger.info(hbaseMapping.getHbaseTable() + " etl completed in: " executor.shutdown();
+ (System.currentTimeMillis() - start) / 1000 + "s!"); } else {
executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg);
etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条"); }
} catch (Exception e) {
logger.error(e.getMessage(), e); logger.info(hbaseMapping.getHbaseTable() + " etl completed in: "
errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); + (System.currentTimeMillis() - start) / 1000 + "s!");
}
etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条");
if (errMsg.isEmpty()) { } catch (Exception e) {
etlResult.setSucceeded(true); logger.error(e.getMessage(), e);
} else { errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); }
}
return etlResult; if (errMsg.isEmpty()) {
} etlResult.setSucceeded(true);
} else {
/** etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
* 执行导入 }
* return etlResult;
* @param ds }
* @param sql
* @param hbaseMapping /**
* @param hbaseTemplate * 执行导入
* @param successCount *
* @param errMsg * @param ds
* @return * @param sql
*/ * @param hbaseMapping
private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping, * @param hbaseTemplate
HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) { * @param successCount
try { * @param errMsg
Util.sqlRS(ds, sql, rs -> { * @return
int i = 1; */
private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping,
try { HbaseTemplate hbaseTemplate, AtomicLong successCount, List<String> errMsg) {
boolean complete = false; try {
List<HRow> rows = new ArrayList<>(); Util.sqlRS(ds, sql, rs -> {
String[] rowKeyColumns = null; int i = 1;
if (hbaseMapping.getRowKey() != null) {
rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); try {
} boolean complete = false;
while (rs.next()) { List<HRow> rows = new ArrayList<>();
int cc = rs.getMetaData().getColumnCount(); String[] rowKeyColumns = null;
int[] jdbcTypes = new int[cc]; if (hbaseMapping.getRowKey() != null) {
Class<?>[] classes = new Class[cc]; rowKeyColumns = hbaseMapping.getRowKey().trim().split(",");
for (int j = 1; j <= cc; j++) { }
int jdbcType = rs.getMetaData().getColumnType(j); while (rs.next()) {
jdbcTypes[j - 1] = jdbcType; int cc = rs.getMetaData().getColumnCount();
classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType); int[] jdbcTypes = new int[cc];
} Class<?>[] classes = new Class[cc];
HRow row = new HRow(); for (int j = 1; j <= cc; j++) {
int jdbcType = rs.getMetaData().getColumnType(j);
if (rowKeyColumns != null) { jdbcTypes[j - 1] = jdbcType;
// 取rowKey字段拼接 classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType);
StringBuilder rowKeyVale = new StringBuilder(); }
for (String rowKeyColumnName : rowKeyColumns) { HRow row = new HRow();
Object obj = rs.getObject(rowKeyColumnName);
if (obj != null) { if (rowKeyColumns != null) {
rowKeyVale.append(obj.toString()); // 取rowKey字段拼接
} StringBuilder rowKeyVale = new StringBuilder();
rowKeyVale.append("|"); for (String rowKeyColumnName : rowKeyColumns) {
} Object obj = rs.getObject(rowKeyColumnName);
int len = rowKeyVale.length(); if (obj != null) {
if (len > 0) { rowKeyVale.append(obj.toString());
rowKeyVale.delete(len - 1, len); }
} rowKeyVale.append("|");
row.setRowKey(Bytes.toBytes(rowKeyVale.toString())); }
} int len = rowKeyVale.length();
if (len > 0) {
for (int j = 1; j <= cc; j++) { rowKeyVale.delete(len - 1, len);
String columnName = rs.getMetaData().getColumnName(j); }
row.setRowKey(Bytes.toBytes(rowKeyVale.toString()));
Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]); }
if (val == null) {
continue; for (int j = 1; j <= cc; j++) {
} String columnName = rs.getMetaData().getColumnName(j);
MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName); Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]);
// 没有配置映射 if (val == null) {
if (columnItem == null) { continue;
String family = hbaseMapping.getFamily(); }
String qualifile = columnName;
if (hbaseMapping.isUppercaseQualifier()) { MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName);
qualifile = qualifile.toUpperCase(); // 没有配置映射
} if (columnItem == null) {
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { String family = hbaseMapping.getFamily();
if (hbaseMapping.getRowKey() == null && j == 1) { String qualifile = columnName;
row.setRowKey(Bytes.toBytes(val.toString())); if (hbaseMapping.isUppercaseQualifier()) {
} else { qualifile = qualifile.toUpperCase();
row.addCell(family, qualifile, Bytes.toBytes(val.toString())); }
} if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { if (hbaseMapping.getRowKey() == null && j == 1) {
Type type = Type.getType(classes[j - 1]); row.setRowKey(Bytes.toBytes(val.toString()));
if (hbaseMapping.getRowKey() == null && j == 1) { } else {
row.setRowKey(TypeUtil.toBytes(val, type)); row.addCell(family, qualifile, Bytes.toBytes(val.toString()));
} else { }
row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
} Type type = Type.getType(classes[j - 1]);
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { if (hbaseMapping.getRowKey() == null && j == 1) {
PhType phType = PhType.getType(classes[j - 1]); row.setRowKey(TypeUtil.toBytes(val, type));
if (hbaseMapping.getRowKey() == null && j == 1) { } else {
row.setRowKey(PhTypeUtil.toBytes(val, phType)); row.addCell(family, qualifile, TypeUtil.toBytes(val, type));
} else { }
row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
} PhType phType = PhType.getType(classes[j - 1]);
} if (hbaseMapping.getRowKey() == null && j == 1) {
} else { row.setRowKey(PhTypeUtil.toBytes(val, phType));
// 如果不需要类型转换 } else {
if (columnItem.getType() == null || "".equals(columnItem.getType())) { row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType));
if (val instanceof java.sql.Date) { }
SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd"); }
val = dateFmt.format((Date) val); } else {
} else if (val instanceof Timestamp) { // 如果不需要类型转换
SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); if (columnItem.getType() == null || "".equals(columnItem.getType())) {
val = datetimeFmt.format((Date) val); if (val instanceof java.sql.Date) {
} SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd");
val = dateFmt.format((Date) val);
byte[] valBytes = Bytes.toBytes(val.toString()); } else if (val instanceof Timestamp) {
if (columnItem.isRowKey()) { SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
if (columnItem.getRowKeyLen() != null) { val = datetimeFmt.format((Date) val);
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); }
row.setRowKey(valBytes);
} else { byte[] valBytes = Bytes.toBytes(val.toString());
row.setRowKey(valBytes); if (columnItem.isRowKey()) {
} if (columnItem.getRowKeyLen() != null) {
} else { valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); row.setRowKey(valBytes);
} } else {
} else { row.setRowKey(valBytes);
if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { }
byte[] valBytes = Bytes.toBytes(val.toString()); } else {
if (columnItem.isRowKey()) { row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
if (columnItem.getRowKeyLen() != null) { }
valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); } else {
} if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) {
row.setRowKey(valBytes); byte[] valBytes = Bytes.toBytes(val.toString());
} else { if (columnItem.isRowKey()) {
row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); if (columnItem.getRowKeyLen() != null) {
} valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val));
} else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { }
Type type = Type.getType(columnItem.getType()); row.setRowKey(valBytes);
if (columnItem.isRowKey()) { } else {
if (columnItem.getRowKeyLen() != null) { row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes);
String v = limitLenNum(columnItem.getRowKeyLen(), val); }
row.setRowKey(Bytes.toBytes(v)); } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) {
} else { Type type = Type.getType(columnItem.getType());
row.setRowKey(TypeUtil.toBytes(val, type)); if (columnItem.isRowKey()) {
} if (columnItem.getRowKeyLen() != null) {
} else { String v = limitLenNum(columnItem.getRowKeyLen(), val);
row.addCell(columnItem.getFamily(), row.setRowKey(Bytes.toBytes(v));
columnItem.getQualifier(), } else {
TypeUtil.toBytes(val, type)); row.setRowKey(TypeUtil.toBytes(val, type));
} }
} else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { } else {
PhType phType = PhType.getType(columnItem.getType()); row.addCell(columnItem.getFamily(),
if (columnItem.isRowKey()) { columnItem.getQualifier(),
row.setRowKey(PhTypeUtil.toBytes(val, phType)); TypeUtil.toBytes(val, type));
} else { }
row.addCell(columnItem.getFamily(), } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) {
columnItem.getQualifier(), PhType phType = PhType.getType(columnItem.getType());
PhTypeUtil.toBytes(val, phType)); if (columnItem.isRowKey()) {
} row.setRowKey(PhTypeUtil.toBytes(val, phType));
} } else {
} row.addCell(columnItem.getFamily(),
} columnItem.getQualifier(),
} PhTypeUtil.toBytes(val, phType));
}
if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空"); }
}
rows.add(row); }
complete = false; }
if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空");
rows.clear();
complete = true; rows.add(row);
} complete = false;
i++; if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) {
successCount.incrementAndGet(); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
if (logger.isDebugEnabled()) { rows.clear();
logger.debug("successful import count:" + successCount.get()); complete = true;
} }
} i++;
successCount.incrementAndGet();
if (!complete && !rows.isEmpty()) { if (logger.isDebugEnabled()) {
hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); logger.debug("successful import count:" + successCount.get());
} }
}
} catch (Exception e) {
logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); if (!complete && !rows.isEmpty()) {
errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage()); hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows);
// throw new RuntimeException(e); }
}
return i; } catch (Exception e) {
}); logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e);
return true; errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage());
} catch (Exception e) { // throw new RuntimeException(e);
logger.error(e.getMessage(), e); }
return false; return i;
} });
} return true;
} catch (Exception e) {
private static String limitLenNum(int len, Object val) { logger.error(e.getMessage(), e);
if (val == null) { return false;
return null; }
} }
if (val instanceof Number) {
return String.format("%0" + len + "d", (Number) ((Number) val).longValue()); private static String limitLenNum(int len, Object val) {
} else if (val instanceof String) { if (val == null) {
return String.format("%0" + len + "d", Long.parseLong((String) val)); return null;
} }
return null; if (val instanceof Number) {
} return String.format("%0" + len + "d", (Number) ((Number) val).longValue());
} } else if (val instanceof String) {
return String.format("%0" + len + "d", Long.parseLong((String) val));
}
return null;
}
}

432
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -1,216 +1,216 @@
package com.alibaba.otter.canal.adapter.launcher.loader; package com.alibaba.otter.canal.adapter.launcher.loader;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.SocketAddress; import java.net.SocketAddress;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.core.env.EnumerablePropertySource; import org.springframework.core.env.EnumerablePropertySource;
import org.springframework.core.env.Environment; import org.springframework.core.env.Environment;
import org.springframework.core.env.PropertySource; import org.springframework.core.env.PropertySource;
import org.springframework.core.env.StandardEnvironment; import org.springframework.core.env.StandardEnvironment;
import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
/** /**
* 外部适配器的加载器 * 外部适配器的加载器
* *
* @version 1.0.0 * @version 1.0.0
*/ */
public class CanalAdapterLoader { public class CanalAdapterLoader {
private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class);
private CanalClientConfig canalClientConfig; private CanalClientConfig canalClientConfig;
private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>(); private Map<String, CanalAdapterWorker> canalWorkers = new HashMap<>();
private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>(); private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
private ExtensionLoader<OuterAdapter> loader; private ExtensionLoader<OuterAdapter> loader;
public CanalAdapterLoader(CanalClientConfig canalClientConfig){ public CanalAdapterLoader(CanalClientConfig canalClientConfig){
this.canalClientConfig = canalClientConfig; this.canalClientConfig = canalClientConfig;
} }
/** /**
* 初始化canal-client * 初始化canal-client
*/ */
public void init() { public void init() {
loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
String canalServerHost = this.canalClientConfig.getCanalServerHost(); String canalServerHost = this.canalClientConfig.getCanalServerHost();
SocketAddress sa = null; SocketAddress sa = null;
if (canalServerHost != null) { if (canalServerHost != null) {
String[] ipPort = canalServerHost.split(":"); String[] ipPort = canalServerHost.split(":");
sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
} }
String zkHosts = this.canalClientConfig.getZookeeperHosts(); String zkHosts = this.canalClientConfig.getZookeeperHosts();
if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client的适配器 // 初始化canal-client的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>(); List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
List<OuterAdapter> canalOutConnectors = new ArrayList<>(); List<OuterAdapter> canalOutConnectors = new ArrayList<>();
for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
loadAdapter(c, canalOutConnectors); loadAdapter(c, canalOutConnectors);
} }
canalOuterAdapterGroups.add(canalOutConnectors); canalOuterAdapterGroups.add(canalOutConnectors);
} }
CanalAdapterWorker worker; CanalAdapterWorker worker;
if (sa != null) { if (sa != null) {
worker = new CanalAdapterWorker(canalClientConfig, worker = new CanalAdapterWorker(canalClientConfig,
canalAdapter.getInstance(), canalAdapter.getInstance(),
sa, sa,
canalOuterAdapterGroups); canalOuterAdapterGroups);
} else if (zkHosts != null) { } else if (zkHosts != null) {
worker = new CanalAdapterWorker(canalClientConfig, worker = new CanalAdapterWorker(canalClientConfig,
canalAdapter.getInstance(), canalAdapter.getInstance(),
zkHosts, zkHosts,
canalOuterAdapterGroups); canalOuterAdapterGroups);
} else { } else {
throw new RuntimeException("No canal server connector found"); throw new RuntimeException("No canal server connector found");
} }
canalWorkers.put(canalAdapter.getInstance(), worker); canalWorkers.put(canalAdapter.getInstance(), worker);
worker.start(); worker.start();
logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
} }
} else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-kafka的适配器 // 初始化canal-client-kafka的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) { for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>(); List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
List<OuterAdapter> canalOuterAdapters = new ArrayList<>(); List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) { for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters); loadAdapter(config, canalOuterAdapters);
} }
canalOuterAdapterGroups.add(canalOuterAdapters); canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
canalClientConfig.getMqServers(), canalClientConfig.getMqServers(),
canalAdapter.getInstance(), canalAdapter.getInstance(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,
canalClientConfig.getFlatMessage()); canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
canalKafkaWorker.start(); canalKafkaWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed", logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId()); canalAdapter.getInstance() + "-" + group.getGroupId());
} }
} }
} else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
// 初始化canal-client-rocketMQ的适配器 // 初始化canal-client-rocketMQ的适配器
for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
for (CanalClientConfig.Group group : canalAdapter.getGroups()) { for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>(); List<List<OuterAdapter>> canalOuterAdapterGroups = new ArrayList<>();
List<OuterAdapter> canalOuterAdapters = new ArrayList<>(); List<OuterAdapter> canalOuterAdapters = new ArrayList<>();
for (OuterAdapterConfig config : group.getOuterAdapters()) { for (OuterAdapterConfig config : group.getOuterAdapters()) {
loadAdapter(config, canalOuterAdapters); loadAdapter(config, canalOuterAdapters);
} }
canalOuterAdapterGroups.add(canalOuterAdapters); canalOuterAdapterGroups.add(canalOuterAdapters);
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
canalClientConfig.getMqServers(), canalClientConfig.getMqServers(),
canalAdapter.getInstance(), canalAdapter.getInstance(),
group.getGroupId(), group.getGroupId(),
canalOuterAdapterGroups, canalOuterAdapterGroups,
canalClientConfig.getAccessKey(), canalClientConfig.getAccessKey(),
canalClientConfig.getSecretKey(), canalClientConfig.getSecretKey(),
canalClientConfig.getFlatMessage()); canalClientConfig.getFlatMessage());
canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
rocketMQWorker.start(); rocketMQWorker.start();
logger.info("Start adapter for canal-client mq topic: {} succeed", logger.info("Start adapter for canal-client mq topic: {} succeed",
canalAdapter.getInstance() + "-" + group.getGroupId()); canalAdapter.getInstance() + "-" + group.getGroupId());
} }
} }
} }
} }
private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) { private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
try { try {
OuterAdapter adapter; OuterAdapter adapter;
adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey())); adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
ClassLoader cl = Thread.currentThread().getContextClassLoader(); ClassLoader cl = Thread.currentThread().getContextClassLoader();
// 替换ClassLoader // 替换ClassLoader
Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
Environment env = (Environment) SpringContext.getBean(Environment.class); Environment env = (Environment) SpringContext.getBean(Environment.class);
Properties evnProperties = null; Properties evnProperties = null;
if (env instanceof StandardEnvironment) { if (env instanceof StandardEnvironment) {
evnProperties = new Properties(); evnProperties = new Properties();
for (PropertySource<?> propertySource : ((StandardEnvironment) env).getPropertySources()) { for (PropertySource<?> propertySource : ((StandardEnvironment) env).getPropertySources()) {
if (propertySource instanceof EnumerablePropertySource) { if (propertySource instanceof EnumerablePropertySource) {
String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames(); String[] names = ((EnumerablePropertySource<?>) propertySource).getPropertyNames();
for (String name : names) { for (String name : names) {
Object val = propertySource.getProperty(name); Object val = propertySource.getProperty(name);
if (val != null) { if (val != null) {
evnProperties.put(name, val); evnProperties.put(name, val);
} }
} }
} }
} }
} }
adapter.init(config, evnProperties); adapter.init(config, evnProperties);
Thread.currentThread().setContextClassLoader(cl); Thread.currentThread().setContextClassLoader(cl);
canalOutConnectors.add(adapter); canalOutConnectors.add(adapter);
logger.info("Load canal adapter: {} succeed", config.getName()); logger.info("Load canal adapter: {} succeed", config.getName());
} catch (Exception e) { } catch (Exception e) {
logger.error("Load canal adapter: {} failed", config.getName(), e); logger.error("Load canal adapter: {} failed", config.getName(), e);
} }
} }
/** /**
* 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
*/ */
public void destroy() { public void destroy() {
if (!canalWorkers.isEmpty()) { if (!canalWorkers.isEmpty()) {
ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
List<Future<Boolean>> futures = new ArrayList<>(); List<Future<Boolean>> futures = new ArrayList<>();
for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
futures.add(stopExecutorService.submit(() -> { futures.add(stopExecutorService.submit(() -> {
canalAdapterWorker.stop(); canalAdapterWorker.stop();
return true; return true;
})); }));
} }
futures.forEach(future -> { futures.forEach(future -> {
try { try {
future.get(); future.get();
} catch (Exception e) { } catch (Exception e) {
// ignore // ignore
} }
}); });
stopExecutorService.shutdown(); stopExecutorService.shutdown();
} }
if (!canalMQWorker.isEmpty()) { if (!canalMQWorker.isEmpty()) {
ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size()); ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
List<Future<Boolean>> futures = new ArrayList<>(); List<Future<Boolean>> futures = new ArrayList<>();
for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) { for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
futures.add(stopMQWorkerService.submit(() -> { futures.add(stopMQWorkerService.submit(() -> {
canalAdapterMQWorker.stop(); canalAdapterMQWorker.stop();
return true; return true;
})); }));
} }
futures.forEach(future -> { futures.forEach(future -> {
try { try {
future.get(); future.get();
} catch (Exception e) { } catch (Exception e) {
// ignore // ignore
} }
}); });
stopMQWorkerService.shutdown(); stopMQWorkerService.shutdown();
} }
logger.info("All canal adapters destroyed"); logger.info("All canal adapters destroyed");
} }
} }

494
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -1,248 +1,246 @@
package com.alibaba.otter.canal.adapter.launcher.monitor.remote; package com.alibaba.otter.canal.adapter.launcher.monitor.remote;
import java.io.FileWriter; import java.io.FileWriter;
import java.sql.Connection; import java.sql.Connection;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.common.utils.CommonUtils; import com.alibaba.otter.canal.common.utils.CommonUtils;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.collect.MapMaker; import com.google.common.collect.MapMaker;
/** /**
* 基于数据库的远程配置装载器 * 基于数据库的远程配置装载器
* *
* @author rewerma 2019-01-25 下午05:20:16 * @author rewerma 2019-01-25 下午05:20:16
* @version 1.0.0 * @version 1.0.0
*/ */
public class DbRemoteConfigLoader implements RemoteConfigLoader { public class DbRemoteConfigLoader implements RemoteConfigLoader {
private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class); private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class);
private DruidDataSource dataSource; private DruidDataSource dataSource;
private static volatile long currentConfigTimestamp = 0; private static volatile long currentConfigTimestamp = 0;
private Map<String, ConfigItem> remoteAdapterConfigs = new MapMaker().makeMap(); private Map<String, ConfigItem> remoteAdapterConfigs = new MapMaker().makeMap();
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2, private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2,
new NamedThreadFactory("remote-adapter-config-scan")); new NamedThreadFactory("remote-adapter-config-scan"));
private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl(); private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl();
public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){ public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){
dataSource = new DruidDataSource(); dataSource = new DruidDataSource();
if (StringUtils.isEmpty(driverName)) { if (StringUtils.isEmpty(driverName)) {
driverName = "com.mysql.jdbc.Driver"; driverName = "com.mysql.jdbc.Driver";
} }
dataSource.setDriverClassName(driverName); dataSource.setDriverClassName(driverName);
dataSource.setUrl(jdbcUrl); dataSource.setUrl(jdbcUrl);
dataSource.setUsername(jdbcUsername); dataSource.setUsername(jdbcUsername);
dataSource.setPassword(jdbcPassword); dataSource.setPassword(jdbcPassword);
dataSource.setInitialSize(1); dataSource.setInitialSize(1);
dataSource.setMinIdle(1); dataSource.setMinIdle(1);
dataSource.setMaxActive(1); dataSource.setMaxActive(1);
dataSource.setMaxWait(60000); dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000); dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000); dataSource.setMinEvictableIdleTimeMillis(300000);
try { try {
dataSource.init(); dataSource.init();
} catch (SQLException e) { } catch (SQLException e) {
throw new RuntimeException(e.getMessage(), e); throw new RuntimeException(e.getMessage(), e);
} }
} }
/** /**
* 加载远程application.yml配置 * 加载远程application.yml配置
*/ */
@Override @Override
public void loadRemoteConfig() { public void loadRemoteConfig() {
try { try {
// 加载远程adapter配置 // 加载远程adapter配置
ConfigItem configItem = getRemoteAdapterConfig(); ConfigItem configItem = getRemoteAdapterConfig();
if (configItem != null) { if (configItem != null) {
if (configItem.getModifiedTime() != currentConfigTimestamp) { if (configItem.getModifiedTime() != currentConfigTimestamp) {
currentConfigTimestamp = configItem.getModifiedTime(); currentConfigTimestamp = configItem.getModifiedTime();
overrideLocalCanalConfig(configItem.getContent()); overrideLocalCanalConfig(configItem.getContent());
logger.info("## Loaded remote adapter config: application.yml"); logger.info("## Loaded remote adapter config: application.yml");
} }
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
/** /**
* 获取远程application.yml配置 * 获取远程application.yml配置
* *
* @return 配置对象 * @return 配置对象
*/ */
private ConfigItem getRemoteAdapterConfig() { private ConfigItem getRemoteAdapterConfig() {
String sql = "select name, content, modified_time from canal_config where id=2"; String sql = "select name, content, modified_time from canal_config where id=2";
try (Connection conn = dataSource.getConnection(); try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement(); Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) { ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) { if (rs.next()) {
ConfigItem configItem = new ConfigItem(); ConfigItem configItem = new ConfigItem();
configItem.setId(2L); configItem.setId(2L);
configItem.setName(rs.getString("name")); configItem.setName(rs.getString("name"));
configItem.setContent(rs.getString("content")); configItem.setContent(rs.getString("content"));
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
return configItem; return configItem;
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
return null; return null;
} }
/** /**
* 覆盖本地application.yml文件 * 覆盖本地application.yml文件
* *
* @param content 文件内容 * @param content 文件内容
*/ */
private void overrideLocalCanalConfig(String content) { private void overrideLocalCanalConfig(String content) {
try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) { try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) {
writer.write(content); writer.write(content);
writer.flush(); writer.flush();
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
/** /**
* 加载adapter配置 * 加载adapter配置
*/ */
@Override @Override
public void loadRemoteAdapterConfigs() { public void loadRemoteAdapterConfigs() {
try { try {
// 加载远程adapter配置 // 加载远程adapter配置
loadModifiedAdapterConfigs(); loadModifiedAdapterConfigs();
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
} }
} }
/** /**
* 加载有变动的adapter配置 * 加载有变动的adapter配置
*/ */
@SuppressWarnings("unchecked") private void loadModifiedAdapterConfigs() {
private void loadModifiedAdapterConfigs() { Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
Map<String, ConfigItem>[] res = new Map[2]; String sql = "select id, category, name, modified_time from canal_adapter_config";
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>(); try (Connection conn = dataSource.getConnection();
String sql = "select id, category, name, modified_time from canal_adapter_config"; Statement stmt = conn.createStatement();
try (Connection conn = dataSource.getConnection(); ResultSet rs = stmt.executeQuery(sql)) {
Statement stmt = conn.createStatement(); while (rs.next()) {
ResultSet rs = stmt.executeQuery(sql)) { ConfigItem configItem = new ConfigItem();
while (rs.next()) { configItem.setId(rs.getLong("id"));
ConfigItem configItem = new ConfigItem(); configItem.setCategory(rs.getString("category"));
configItem.setId(rs.getLong("id")); configItem.setName(rs.getString("name"));
configItem.setCategory(rs.getString("category")); configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime());
configItem.setName(rs.getString("name")); remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem);
configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); }
remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem); } catch (Exception e) {
} logger.error(e.getMessage(), e);
} catch (Exception e) { }
logger.error(e.getMessage(), e);
} if (!remoteConfigStatus.isEmpty()) {
List<Long> changedIds = new ArrayList<>();
if (!remoteConfigStatus.isEmpty()) {
List<Long> changedIds = new ArrayList<>(); for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) {
ConfigItem currentConfig = remoteAdapterConfigs
for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) { .get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName());
ConfigItem currentConfig = remoteAdapterConfigs if (currentConfig == null) {
.get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName()); // 新增
if (currentConfig == null) { changedIds.add(remoteConfigStat.getId());
// 新增 } else {
changedIds.add(remoteConfigStat.getId()); // 修改
} else { if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) {
// 修改 changedIds.add(remoteConfigStat.getId());
if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) { }
changedIds.add(remoteConfigStat.getId()); }
} }
} if (!changedIds.isEmpty()) {
} String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in ("
if (!changedIds.isEmpty()) { + Joiner.on(",").join(changedIds) + ")";
String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in (" try (Connection conn = dataSource.getConnection();
+ Joiner.on(",").join(changedIds) + ")"; Statement stmt = conn.createStatement();
try (Connection conn = dataSource.getConnection(); ResultSet rs = stmt.executeQuery(contentsSql)) {
Statement stmt = conn.createStatement(); while (rs.next()) {
ResultSet rs = stmt.executeQuery(contentsSql)) { ConfigItem configItemNew = new ConfigItem();
while (rs.next()) { configItemNew.setId(rs.getLong("id"));
ConfigItem configItemNew = new ConfigItem(); configItemNew.setCategory(rs.getString("category"));
configItemNew.setId(rs.getLong("id")); configItemNew.setName(rs.getString("name"));
configItemNew.setCategory(rs.getString("category")); configItemNew.setContent(rs.getString("content"));
configItemNew.setName(rs.getString("name")); configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime());
configItemNew.setContent(rs.getString("content"));
configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime()); remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(),
configItemNew);
remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(), remoteAdapterMonitor.onModify(configItemNew);
configItemNew); }
remoteAdapterMonitor.onModify(configItemNew);
} } catch (Exception e) {
logger.error(e.getMessage(), e);
} catch (Exception e) { }
logger.error(e.getMessage(), e); }
} }
}
} for (ConfigItem configItem : remoteAdapterConfigs.values()) {
if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) {
for (ConfigItem configItem : remoteAdapterConfigs.values()) { // 删除
if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) { remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName());
// 删除 remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName());
remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName()); }
remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName()); }
} }
}
} /**
* 启动监听数据库变化
/** */
* 启动监听数据库变化 @Override
*/ public void startMonitor() {
@Override // 监听application.yml变化
public void startMonitor() { executor.scheduleWithFixedDelay(() -> {
// 监听application.yml变化 try {
executor.scheduleWithFixedDelay(() -> { loadRemoteConfig();
try { } catch (Throwable e) {
loadRemoteConfig(); logger.error("scan remote application.yml failed", e);
} catch (Throwable e) { }
logger.error("scan remote application.yml failed", e); }, 10, 3, TimeUnit.SECONDS);
}
}, 10, 3, TimeUnit.SECONDS); // 监听adapter变化
executor.scheduleWithFixedDelay(() -> {
// 监听adapter变化 try {
executor.scheduleWithFixedDelay(() -> { loadRemoteAdapterConfigs();
try { } catch (Throwable e) {
loadRemoteAdapterConfigs(); logger.error("scan remote adapter configs failed", e);
} catch (Throwable e) { }
logger.error("scan remote adapter configs failed", e); }, 10, 3, TimeUnit.SECONDS);
} }
}, 10, 3, TimeUnit.SECONDS);
} /**
* 销毁
/** */
* 销毁 @Override
*/ public void destroy() {
@Override executor.shutdownNow();
public void destroy() { try {
executor.shutdownNow(); dataSource.close();
try { } catch (Exception e) {
dataSource.close(); logger.error(e.getMessage(), e);
} catch (Exception e) { }
logger.error(e.getMessage(), e); }
} }
}
}

104
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java
View File

@@ -1,53 +1,51 @@
package com.alibaba.otter.canal.client.adapter.rdb.config; package com.alibaba.otter.canal.client.adapter.rdb.config;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import org.yaml.snakeyaml.Yaml; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /**
* RDB表映射配置加载器
/** *
* RDB表映射配置加载器 * @author rewerma 2018-11-07 下午02:41:34
* * @version 1.0.0
* @author rewerma 2018-11-07 下午02:41:34 */
* @version 1.0.0 public class ConfigLoader {
*/
public class ConfigLoader { private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class);
private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); /**
* 加载RDB表映射配置
/** *
* 加载RDB表映射配置 * @return 配置名/配置文件名--对象
* */
* @return 配置名/配置文件名--对象 public static Map<String, MappingConfig> load(Properties envProperties) {
*/ logger.info("## Start loading rdb mapping config ... ");
public static Map<String, MappingConfig> load(Properties envProperties) {
logger.info("## Start loading rdb mapping config ... "); Map<String, MappingConfig> result = new LinkedHashMap<>();
Map<String, MappingConfig> result = new LinkedHashMap<>(); Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("rdb");
configContentMap.forEach((fileName, content) -> {
Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("rdb"); MappingConfig config = YmlConfigBinder
configContentMap.forEach((fileName, content) -> { .bindYmlToObj(null, content, MappingConfig.class, null, envProperties);
MappingConfig config = YmlConfigBinder if (config == null) {
.bindYmlToObj(null, content, MappingConfig.class, null, envProperties); return;
if (config == null) { }
return; try {
} config.validate();
try { } catch (Exception e) {
config.validate(); throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
} catch (Exception e) { }
throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); result.put(fileName, config);
} });
result.put(fileName, config);
}); logger.info("## Rdb mapping config loaded");
return result;
logger.info("## Rdb mapping config loaded"); }
return result; }
}
}

339
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java
View File

@@ -1,170 +1,169 @@
package com.alibaba.otter.canal.client.adapter.rdb.monitor; package com.alibaba.otter.canal.client.adapter.rdb.monitor;
import java.io.File; import java.io.File;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; import org.apache.commons.io.monitor.FileAlterationMonitor;
import org.apache.commons.io.monitor.FileAlterationMonitor; import org.apache.commons.io.monitor.FileAlterationObserver;
import org.apache.commons.io.monitor.FileAlterationObserver; import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.StringUtils; import org.slf4j.Logger;
import org.slf4j.Logger; import org.slf4j.LoggerFactory;
import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter;
import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig;
import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; import com.alibaba.otter.canal.client.adapter.support.Util;
import com.alibaba.otter.canal.client.adapter.support.Util;
public class RdbConfigMonitor {
public class RdbConfigMonitor {
private static final Logger logger = LoggerFactory.getLogger(RdbConfigMonitor.class);
private static final Logger logger = LoggerFactory.getLogger(RdbConfigMonitor.class);
private static final String adapterName = "rdb";
private static final String adapterName = "rdb";
private String key;
private String key;
private RdbAdapter rdbAdapter;
private RdbAdapter rdbAdapter;
private FileAlterationMonitor fileMonitor;
private FileAlterationMonitor fileMonitor;
public void init(String key, RdbAdapter rdbAdapter) {
public void init(String key, RdbAdapter rdbAdapter) { this.key = key;
this.key = key; this.rdbAdapter = rdbAdapter;
this.rdbAdapter = rdbAdapter; File confDir = Util.getConfDirPath(adapterName);
File confDir = Util.getConfDirPath(adapterName); try {
try { FileAlterationObserver observer = new FileAlterationObserver(confDir,
FileAlterationObserver observer = new FileAlterationObserver(confDir, FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); FileListener listener = new FileListener();
FileListener listener = new FileListener(); observer.addListener(listener);
observer.addListener(listener); fileMonitor = new FileAlterationMonitor(3000, observer);
fileMonitor = new FileAlterationMonitor(3000, observer); fileMonitor.start();
fileMonitor.start();
} catch (Exception e) {
} catch (Exception e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(), e); }
} }
}
public void destroy() {
public void destroy() { try {
try { fileMonitor.stop();
fileMonitor.stop(); } catch (Exception e) {
} catch (Exception e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(), e); }
} }
}
private class FileListener extends FileAlterationListenerAdaptor {
private class FileListener extends FileAlterationListenerAdaptor {
@Override
@Override public void onFileCreate(File file) {
public void onFileCreate(File file) { super.onFileCreate(file);
super.onFileCreate(file); try {
try { // 加载新增的配置文件
// 加载新增的配置文件 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); config.validate();
config.validate(); if ((key == null && config.getOuterAdapterKey() == null)
if ((key == null && config.getOuterAdapterKey() == null) || (key != null && key.equals(config.getOuterAdapterKey()))) {
|| (key != null && key.equals(config.getOuterAdapterKey()))) { addConfigToCache(file, config);
addConfigToCache(file, config);
logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName());
logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName()); }
} } catch (Exception e) {
} catch (Exception e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(), e); }
} }
}
@Override
@Override public void onFileChange(File file) {
public void onFileChange(File file) { super.onFileChange(file);
super.onFileChange(file);
try {
try { if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { // 加载配置文件
// 加载配置文件 String configContent = MappingConfigsLoader
String configContent = MappingConfigsLoader .loadConfig(adapterName + File.separator + file.getName());
.loadConfig(adapterName + File.separator + file.getName()); if (configContent == null) {
if (configContent == null) { onFileDelete(file);
onFileDelete(file); return;
return; }
} MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class);
MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); config.validate();
config.validate(); if ((key == null && config.getOuterAdapterKey() == null)
if ((key == null && config.getOuterAdapterKey() == null) || (key != null && key.equals(config.getOuterAdapterKey()))) {
|| (key != null && key.equals(config.getOuterAdapterKey()))) { if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { deleteConfigFromCache(file);
deleteConfigFromCache(file); }
} addConfigToCache(file, config);
addConfigToCache(file, config); } else {
} else { // 不能修改outerAdapterKey
// 不能修改outerAdapterKey throw new RuntimeException("Outer adapter key not allowed modify");
throw new RuntimeException("Outer adapter key not allowed modify"); }
} logger.info("Change a rdb mapping config: {} of canal adapter", file.getName());
logger.info("Change a rdb mapping config: {} of canal adapter", file.getName()); }
} } catch (Exception e) {
} catch (Exception e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(), e); }
} }
}
@Override
@Override public void onFileDelete(File file) {
public void onFileDelete(File file) { super.onFileDelete(file);
super.onFileDelete(file);
try {
try { if (rdbAdapter.getRdbMapping().containsKey(file.getName())) {
if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { deleteConfigFromCache(file);
deleteConfigFromCache(file);
logger.info("Delete a rdb mapping config: {} of canal adapter", file.getName());
logger.info("Delete a rdb mapping config: {} of canal adapter", file.getName()); }
} } catch (Exception e) {
} catch (Exception e) { logger.error(e.getMessage(), e);
logger.error(e.getMessage(), e); }
} }
}
private void addConfigToCache(File file, MappingConfig mappingConfig) {
private void addConfigToCache(File file, MappingConfig mappingConfig) { if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
if (mappingConfig == null || mappingConfig.getDbMapping() == null) { return;
return; }
} rdbAdapter.getRdbMapping().put(file.getName(), mappingConfig);
rdbAdapter.getRdbMapping().put(file.getName(), mappingConfig); if (!mappingConfig.getDbMapping().getMirrorDb()) {
if (!mappingConfig.getDbMapping().getMirrorDb()) { Map<String, MappingConfig> configMap = rdbAdapter.getMappingConfigCache()
Map<String, MappingConfig> configMap = rdbAdapter.getMappingConfigCache() .computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
.computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping().getDatabase() + "."
+ mappingConfig.getDbMapping().getDatabase() + "." + mappingConfig.getDbMapping().getTable(),
+ mappingConfig.getDbMapping().getTable(), k1 -> new HashMap<>());
k1 -> new HashMap<>()); configMap.put(file.getName(), mappingConfig);
configMap.put(file.getName(), mappingConfig); } else {
} else { Map<String, MirrorDbConfig> mirrorDbConfigCache = rdbAdapter.getMirrorDbConfigCache();
Map<String, MirrorDbConfig> mirrorDbConfigCache = rdbAdapter.getMirrorDbConfigCache(); mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "."
mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + mappingConfig.getDbMapping().getDatabase(),
+ mappingConfig.getDbMapping().getDatabase(), MirrorDbConfig.create(file.getName(), mappingConfig));
MirrorDbConfig.create(file.getName(), mappingConfig)); }
} }
}
private void deleteConfigFromCache(File file) {
private void deleteConfigFromCache(File file) { MappingConfig mappingConfig = rdbAdapter.getRdbMapping().remove(file.getName());
MappingConfig mappingConfig = rdbAdapter.getRdbMapping().remove(file.getName());
if (mappingConfig == null || mappingConfig.getDbMapping() == null) {
if (mappingConfig == null || mappingConfig.getDbMapping() == null) { return;
return; }
} if (!mappingConfig.getDbMapping().getMirrorDb()) {
if (!mappingConfig.getDbMapping().getMirrorDb()) { for (Map<String, MappingConfig> configMap : rdbAdapter.getMappingConfigCache().values()) {
for (Map<String, MappingConfig> configMap : rdbAdapter.getMappingConfigCache().values()) { if (configMap != null) {
if (configMap != null) { configMap.remove(file.getName());
configMap.remove(file.getName()); }
} }
} } else {
} else { rdbAdapter.getMirrorDbConfigCache().forEach((key, mirrorDbConfig) -> {
rdbAdapter.getMirrorDbConfigCache().forEach((key, mirrorDbConfig) -> { if (mirrorDbConfig.getFileName().equals(file.getName())) {
if (mirrorDbConfig.getFileName().equals(file.getName())) { rdbAdapter.getMirrorDbConfigCache().remove(key);
rdbAdapter.getMirrorDbConfigCache().remove(key); }
} });
}); }
}
}
} }
} }
}

1005
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

File diff suppressed because it is too large Load Diff

651
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -1,326 +1,325 @@
package com.alibaba.otter.canal.client.kafka; package com.alibaba.otter.canal.client.kafka;
import java.util.*; import java.util.*;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.google.common.collect.MapMaker; import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSON; import com.alibaba.otter.canal.client.CanalMQConnector;
import com.alibaba.otter.canal.client.CanalMQConnector; import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.alibaba.otter.canal.protocol.exception.CanalClientException; import com.google.common.collect.Lists;
import com.google.common.collect.Lists;
/**
/** * canal kafka 数据操作客户端
* canal kafka 数据操作客户端 *
* * <pre>
* <pre> * 注意点：
* 注意点： * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
* 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get * </pre>
* </pre> *
* * @author machengyuan @ 2018-6-12
* @author machengyuan @ 2018-6-12 * @version 1.1.1
* @version 1.1.1 */
*/ public class KafkaCanalConnector implements CanalMQConnector {
public class KafkaCanalConnector implements CanalMQConnector {
protected KafkaConsumer<String, Message> kafkaConsumer;
protected KafkaConsumer<String, Message> kafkaConsumer; protected KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费
protected KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费 protected String topic;
protected String topic; protected Integer partition;
protected Integer partition; protected Properties properties;
protected Properties properties; protected volatile boolean connected = false;
protected volatile boolean connected = false; protected volatile boolean running = false;
protected volatile boolean running = false; protected boolean flatMessage;
protected boolean flatMessage;
private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
private Map<Integer, Long> currentOffsets = new ConcurrentHashMap<>();
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize,
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize, boolean flatMessage){
boolean flatMessage){ this.topic = topic;
this.topic = topic; this.partition = partition;
this.partition = partition; this.flatMessage = flatMessage;
this.flatMessage = flatMessage;
properties = new Properties();
properties = new Properties(); properties.put("bootstrap.servers", servers);
properties.put("bootstrap.servers", servers); properties.put("group.id", groupId);
properties.put("group.id", groupId); properties.put("enable.auto.commit", false);
properties.put("enable.auto.commit", false); properties.put("auto.commit.interval.ms", "1000");
properties.put("auto.commit.interval.ms", "1000"); properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读 properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 properties.put("session.timeout.ms", "30000"); // 默认为30秒
properties.put("session.timeout.ms", "30000"); // 默认为30秒 if (batchSize == null) {
if (batchSize == null) { batchSize = 100;
batchSize = 100; }
} properties.put("max.poll.records", batchSize.toString());
properties.put("max.poll.records", batchSize.toString()); properties.put("key.deserializer", StringDeserializer.class.getName());
properties.put("key.deserializer", StringDeserializer.class.getName()); if (!flatMessage) {
if (!flatMessage) { properties.put("value.deserializer", MessageDeserializer.class.getName());
properties.put("value.deserializer", MessageDeserializer.class.getName()); } else {
} else { properties.put("value.deserializer", StringDeserializer.class.getName());
properties.put("value.deserializer", StringDeserializer.class.getName()); }
} }
}
/**
/** * 打开连接
* 打开连接 */
*/ @Override
@Override public void connect() {
public void connect() { if (connected) {
if (connected) { return;
return; }
}
connected = true;
connected = true; if (kafkaConsumer == null && !flatMessage) {
if (kafkaConsumer == null && !flatMessage) { kafkaConsumer = new KafkaConsumer<String, Message>(properties);
kafkaConsumer = new KafkaConsumer<String, Message>(properties);
}
} if (kafkaConsumer2 == null && flatMessage) {
if (kafkaConsumer2 == null && flatMessage) { kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
kafkaConsumer2 = new KafkaConsumer<String, String>(properties); }
} }
}
/**
/** * 关闭链接
* 关闭链接 */
*/ @Override
@Override public void disconnect() {
public void disconnect() { if (kafkaConsumer != null) {
if (kafkaConsumer != null) { kafkaConsumer.close();
kafkaConsumer.close(); kafkaConsumer = null;
kafkaConsumer = null; }
} if (kafkaConsumer2 != null) {
if (kafkaConsumer2 != null) { kafkaConsumer2.close();
kafkaConsumer2.close(); kafkaConsumer2 = null;
kafkaConsumer2 = null; }
}
connected = false;
connected = false; }
}
protected void waitClientRunning() {
protected void waitClientRunning() { running = true;
running = true; }
}
@Override
@Override public boolean checkValid() {
public boolean checkValid() { return true;// 默认都放过
return true;// 默认都放过 }
}
/**
/** * 订阅topic
* 订阅topic */
*/ @Override
@Override public void subscribe() {
public void subscribe() { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return;
return; }
}
if (partition == null) {
if (partition == null) { if (kafkaConsumer != null) {
if (kafkaConsumer != null) { kafkaConsumer.subscribe(Collections.singletonList(topic));
kafkaConsumer.subscribe(Collections.singletonList(topic)); }
} if (kafkaConsumer2 != null) {
if (kafkaConsumer2 != null) { kafkaConsumer2.subscribe(Collections.singletonList(topic));
kafkaConsumer2.subscribe(Collections.singletonList(topic)); }
} } else {
} else { TopicPartition topicPartition = new TopicPartition(topic, partition);
TopicPartition topicPartition = new TopicPartition(topic, partition); if (kafkaConsumer != null) {
if (kafkaConsumer != null) { kafkaConsumer.assign(Collections.singletonList(topicPartition));
kafkaConsumer.assign(Collections.singletonList(topicPartition)); }
} if (kafkaConsumer2 != null) {
if (kafkaConsumer2 != null) { kafkaConsumer2.assign(Collections.singletonList(topicPartition));
kafkaConsumer2.assign(Collections.singletonList(topicPartition)); }
} }
} }
}
/**
/** * 取消订阅
* 取消订阅 */
*/ @Override
@Override public void unsubscribe() {
public void unsubscribe() { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return;
return; }
}
if (kafkaConsumer != null) {
if (kafkaConsumer != null) { kafkaConsumer.unsubscribe();
kafkaConsumer.unsubscribe(); }
} if (kafkaConsumer2 != null) {
if (kafkaConsumer2 != null) { kafkaConsumer2.unsubscribe();
kafkaConsumer2.unsubscribe(); }
} }
}
@Override
@Override public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException {
public List<Message> getList(Long timeout, TimeUnit unit) throws CanalClientException { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return Lists.newArrayList();
return Lists.newArrayList(); }
}
List<Message> messages = getListWithoutAck(timeout, unit);
List<Message> messages = getListWithoutAck(timeout, unit); if (messages != null && !messages.isEmpty()) {
if (messages != null && !messages.isEmpty()) { this.ack();
this.ack(); }
} return messages;
return messages; }
}
@Override
@Override public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
public List<Message> getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return Lists.newArrayList();
return Lists.newArrayList(); }
}
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout));
currentOffsets.clear();
currentOffsets.clear(); for (TopicPartition topicPartition : records.partitions()) {
for (TopicPartition topicPartition : records.partitions()) { currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition));
currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition)); }
}
if (!records.isEmpty()) {
if (!records.isEmpty()) { List<Message> messages = new ArrayList<>();
List<Message> messages = new ArrayList<>(); for (ConsumerRecord<String, Message> record : records) {
for (ConsumerRecord<String, Message> record : records) { messages.add(record.value());
messages.add(record.value()); }
} return messages;
return messages; }
} return Lists.newArrayList();
return Lists.newArrayList(); }
}
@Override
@Override public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException {
public List<FlatMessage> getFlatList(Long timeout, TimeUnit unit) throws CanalClientException { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return Lists.newArrayList();
return Lists.newArrayList(); }
}
List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit);
List<FlatMessage> messages = getFlatListWithoutAck(timeout, unit); if (messages != null && !messages.isEmpty()) {
if (messages != null && !messages.isEmpty()) { this.ack();
this.ack(); }
} return messages;
return messages; }
}
@Override
@Override public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException {
public List<FlatMessage> getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return Lists.newArrayList();
return Lists.newArrayList(); }
}
ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
currentOffsets.clear();
currentOffsets.clear(); for (TopicPartition topicPartition : records.partitions()) {
for (TopicPartition topicPartition : records.partitions()) { currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition));
currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition)); }
}
if (!records.isEmpty()) {
if (!records.isEmpty()) { List<FlatMessage> flatMessages = new ArrayList<>();
List<FlatMessage> flatMessages = new ArrayList<>(); for (ConsumerRecord<String, String> record : records) {
for (ConsumerRecord<String, String> record : records) { String flatMessageJson = record.value();
String flatMessageJson = record.value(); FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); flatMessages.add(flatMessage);
flatMessages.add(flatMessage); }
}
return flatMessages;
return flatMessages; }
} return Lists.newArrayList();
return Lists.newArrayList(); }
}
@Override
@Override public void rollback() {
public void rollback() { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return;
return; }
} // 回滚所有分区
// 回滚所有分区 if (kafkaConsumer != null) {
if (kafkaConsumer != null) { for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) { kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); }
} }
} if (kafkaConsumer2 != null) {
if (kafkaConsumer2 != null) { for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) { kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1);
kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); }
} }
} }
}
/**
/** * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败
* 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败 */
*/ @Override
@Override public void ack() {
public void ack() { waitClientRunning();
waitClientRunning(); if (!running) {
if (!running) { return;
return; }
}
if (kafkaConsumer != null) {
if (kafkaConsumer != null) { kafkaConsumer.commitSync();
kafkaConsumer.commitSync(); }
} if (kafkaConsumer2 != null) {
if (kafkaConsumer2 != null) { kafkaConsumer2.commitSync();
kafkaConsumer2.commitSync(); }
} }
}
@Override
@Override public void subscribe(String filter) throws CanalClientException {
public void subscribe(String filter) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
@Override
@Override public Message get(int batchSize) throws CanalClientException {
public Message get(int batchSize) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
@Override
@Override public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
@Override
@Override public Message getWithoutAck(int batchSize) throws CanalClientException {
public Message getWithoutAck(int batchSize) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
@Override
@Override public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
@Override
@Override public void ack(long batchId) throws CanalClientException {
public void ack(long batchId) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
@Override
@Override public void rollback(long batchId) throws CanalClientException {
public void rollback(long batchId) throws CanalClientException { throw new CanalClientException("mq not support this method");
throw new CanalClientException("mq not support this method"); }
}
/**
/** * 重新设置sessionTime
* 重新设置sessionTime *
* * @param timeout
* @param timeout * @param unit
* @param unit */
*/ public void setSessionTimeout(Long timeout, TimeUnit unit) {
public void setSessionTimeout(Long timeout, TimeUnit unit) { long t = unit.toMillis(timeout);
long t = unit.toMillis(timeout); properties.put("request.timeout.ms", String.valueOf(t + 60000));
properties.put("request.timeout.ms", String.valueOf(t + 60000)); properties.put("session.timeout.ms", String.valueOf(t));
properties.put("session.timeout.ms", String.valueOf(t)); }
}
}
}

67
client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java
View File

@@ -1,32 +1,35 @@
package com.alibaba.otter.canal.client.kafka.protocol; package com.alibaba.otter.canal.client.kafka.protocol;
import com.alibaba.otter.canal.protocol.FlatMessage; import com.alibaba.otter.canal.protocol.FlatMessage;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
/** /**
* 消息对象（Kafka） * 消息对象（Kafka）
* *
* @Author panjianping * @Author panjianping
* @Email ipanjianping@qq.com * @Email ipanjianping@qq.com
* @Date 2018/12/17 * @Date 2018/12/17
*/ */
public class KafkaFlatMessage extends FlatMessage { public class KafkaFlatMessage extends FlatMessage {
/**
* Kafka 消息 offset private static final long serialVersionUID = 5748024400508080710L;
*/
private long offset; /**
* Kafka 消息 offset
public KafkaFlatMessage(FlatMessage message, long offset) { */
super(message.getId()); private long offset;
BeanUtils.copyProperties(message, this);
this.offset = offset; public KafkaFlatMessage(FlatMessage message, long offset){
} super(message.getId());
BeanUtils.copyProperties(message, this);
public long getOffset() { this.offset = offset;
return offset; }
}
public long getOffset() {
public void setOffset(long offset) { return offset;
this.offset = offset; }
}
} public void setOffset(long offset) {
this.offset = offset;
}
}

68
client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java
View File

@@ -1,33 +1,35 @@
package com.alibaba.otter.canal.client.kafka.protocol; package com.alibaba.otter.canal.client.kafka.protocol;
import com.alibaba.otter.canal.protocol.Message; import com.alibaba.otter.canal.protocol.Message;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
/** /**
* 消息对象（Kafka） * 消息对象（Kafka）
* *
* @Author panjianping * @Author panjianping
* @Email ipanjianping@qq.com * @Email ipanjianping@qq.com
* @Date 2018/12/17 * @Date 2018/12/17
*/ */
public class KafkaMessage extends Message { public class KafkaMessage extends Message {
/**
* Kafka 消息 offset private static final long serialVersionUID = -293120358490119447L;
*/
private long offset; /**
* Kafka 消息 offset
public KafkaMessage(Message message, long offset) { */
super(message.getId()); private long offset;
BeanUtils.copyProperties(message, this);
this.offset = offset; public KafkaMessage(Message message, long offset){
} super(message.getId());
BeanUtils.copyProperties(message, this);
this.offset = offset;
public long getOffset() { }
return offset;
} public long getOffset() {
return offset;
public void setOffset(long offset) { }
this.offset = offset;
} public void setOffset(long offset) {
} this.offset = offset;
}
}