chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge canal repo master

Browse Source
This commit is contained in:
duhengforever
2018-09-27 17:09:00 +08:00
parent a37a7ff142 ff592758cf
commit 5ae16b1c7c
73 changed files with 12936 additions and 10729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/CanalOuterAdapter.java
View File

@@ -1,7 +1,9 @@
package com.alibaba.otter.canal.client.adapter;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -23,9 +25,11 @@ public interface CanalOuterAdapter {
/**
* 往适配器中写入数据
*
* @param message message数据包
* @param dml 数据包
*/
void writeOut(Message message);
void writeOut(Dml dml);
// void writeOut(FlatMessage flatMessage);
/**
* 外部适配器销毁接口

30
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java
View File

@@ -22,6 +22,9 @@ public class CanalClientConfig {
private List<MQTopic> mqTopics;
private Boolean flatMessage = true;
private List<CanalInstance> canalInstances;
public String getCanalServerHost() {
@@ -60,6 +63,14 @@ public class CanalClientConfig {
return mqTopics;
}
public Boolean getFlatMessage() {
return flatMessage;
}
public void setFlatMessage(Boolean flatMessage) {
this.flatMessage = flatMessage;
}
public void setMqTopics(List<MQTopic> mqTopics) {
this.mqTopics = mqTopics;
}
@@ -166,25 +177,6 @@ public class CanalClientConfig {
this.outAdapters = outAdapters;
}
// public List<Adaptor> getAdapters() {
// return adapters;
// }
//
// public void setAdapters(List<Adaptor> adapters) {
// this.adapters = adapters;
// }
}
// public static class Adaptor {
// private List<CanalOuterAdapterConfiguration> outAdapters;
//
// public List<CanalOuterAdapterConfiguration> getOutAdapters() {
// return outAdapters;
// }
//
// public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters)
// {
// this.outAdapters = outAdapters;
// }
// }
}

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java
View File

@@ -21,7 +21,7 @@ public class JdbcTypeUtil {
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) {
if (value == null || value.equals("")) {
return null;
}
@@ -96,7 +96,7 @@ public class JdbcTypeUtil {
}
return res;
} catch (Exception e) {
logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
logger.error("table: {} column: {}, failed convert type {} to {}", columnName, value, sqlType);
return value;
}
}

58
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
View File

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import java.util.*;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -63,8 +64,7 @@ public class MessageUtil {
for (CanalEntry.Column column : columns) {
row.put(column.getName(),
JdbcTypeUtil.typeConvert(dml.getTable(),
column.getName(),
JdbcTypeUtil.typeConvert(column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
@@ -82,8 +82,7 @@ public class MessageUtil {
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
rowOld.put(column.getName(),
JdbcTypeUtil.typeConvert(dml.getTable(),
column.getName(),
JdbcTypeUtil.typeConvert(column.getName(),
column.getValue(),
column.getSqlType(),
column.getMysqlType()));
@@ -106,6 +105,57 @@ public class MessageUtil {
}
}
public static Dml flatMessage2Dml(FlatMessage flatMessage) {
if (flatMessage == null) {
return null;
}
Dml dml = new Dml();
dml.setDatabase(flatMessage.getDatabase());
dml.setTable(flatMessage.getTable());
dml.setType(flatMessage.getType());
dml.setTs(flatMessage.getTs());
dml.setSql(flatMessage.getSql());
if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
throw new RuntimeException("SqlType or mysqlType is null");
}
List<Map<String, String>> data = flatMessage.getData();
if (data != null) {
dml.setData(changeRows(data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
}
List<Map<String, String>> old = flatMessage.getOld();
if (old != null) {
dml.setOld(changeRows(old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
}
return dml;
}
private static List<Map<String, Object>> changeRows(List<Map<String, String>> rows, Map<String, Integer> sqlTypes,
Map<String, String> mysqlTypes) {
List<Map<String, Object>> result = new ArrayList<>();
for (Map<String, String> row : rows) {
Map<String, Object> resultRow = new LinkedHashMap<>();
for (Map.Entry<String, String> entry : row.entrySet()) {
String columnName = entry.getKey();
String columnValue = entry.getValue();
Integer sqlType = sqlTypes.get(columnName);
if (sqlType == null) {
continue;
}
String mysqlType = mysqlTypes.get(columnName);
if (mysqlType == null) {
continue;
}
Object finalValue = JdbcTypeUtil.typeConvert(columnName, columnValue, sqlType, mysqlType);
resultRow.put(columnName, finalValue);
}
result.add(resultRow);
}
return result;
}
public interface Consumer<T> {
void accept(T t);

23
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java
View File

@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import com.alibaba.otter.canal.protocol.FlatMessage;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -86,20 +87,14 @@ public class HbaseAdapter implements CanalOuterAdapter {
}
@Override
public void writeOut(Message message) {
MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
@Override
public void accept(Dml dml) {
if (dml == null) {
return;
}
String database = dml.getDatabase();
String table = dml.getTable();
MappingConfig config = mappingConfigCache.get(database + "-" + table);
hbaseSyncService.sync(config, dml);
}
});
public void writeOut(Dml dml) {
if (dml == null) {
return;
}
String database = dml.getDatabase();
String table = dml.getTable();
MappingConfig config = mappingConfigCache.get(database + "-" + table);
hbaseSyncService.sync(config, dml);
}
@Override

15
client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java
View File

@@ -6,9 +6,7 @@ import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.protocol.Message;
/**
* 外部适配器示例
@@ -22,20 +20,13 @@ public class LoggerAdapterExample implements CanalOuterAdapter {
private Logger logger = LoggerFactory.getLogger(this.getClass());
@Override
public void writeOut(Message message) {
// 直接输出日志信息
MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
public void init(CanalOuterAdapterConfiguration configuration) {
@Override
public void accept(Dml dml) {
logger.info(dml.toString());
}
});
}
@Override
public void init(CanalOuterAdapterConfiguration configuration) {
public void writeOut(Dml dml) {
logger.info(dml.toString());
}
@Override