chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

整理代码

Browse Source
This commit is contained in:
rewer
2019-02-07 13:49:35 +08:00
parent ad62275cd7
commit 0c5f5c128f
14 changed files with 3253 additions and 3243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java
View File

@@ -67,8 +67,8 @@ public class YmlConfigBinder {
}
/**
* 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers}
* 在envProperties中有属性 zookeeper.servers:
* 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers:
* ${zookeeper.servers} 在envProperties中有属性 zookeeper.servers:
* 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为
* zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中
* zkServers 属性是 List<String> 对象, 则会自动映射成List
@@ -90,7 +90,7 @@ public class YmlConfigBinder {
}
YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader();
Resource configResource = new ByteArrayResource(contentBytes);
PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null);
PropertySource<?> propertySource = propertySourceLoader.load("manualBindConfig", configResource, null);
if (propertySource == null) {
return null;
@@ -102,15 +102,15 @@ public class YmlConfigBinder {
prefix = prefix + ".";
}
properties.putAll((Map) propertySource.getSource());
properties.putAll((Map<?, ?>) propertySource.getSource());
if (baseProperties != null) {
baseProperties.putAll(properties);
properties = baseProperties;
}
for (Object o : ((Map) propertySource.getSource()).entrySet()) {
Map.Entry entry = (Map.Entry) o;
for (Object o : ((Map<?, ?>) propertySource.getSource()).entrySet()) {
Map.Entry<?, ?> entry = (Map.Entry<?, ?>) o;
String key = (String) entry.getKey();
Object value = entry.getValue();

1
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java
View File

@@ -6,7 +6,6 @@ import java.io.InputStream;
import java.util.Arrays;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.InputStreamSource;
import org.springframework.core.io.Resource;
import org.springframework.util.Assert;

2
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java
View File

@@ -80,7 +80,7 @@ public class SchemaItem {
columnFields = new LinkedHashMap<>();
getSelectFields()
.forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> {
TableItem tableItem = getAliasTableItems().get(columnItem.getOwner());
// TableItem tableItem = getAliasTableItems().get(columnItem.getOwner());
// if (!tableItem.isSubQuery()) {
List<FieldItem> fieldItems = columnFields.computeIfAbsent(
columnItem.getOwner() + "." + columnItem.getColumnName(),

2
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java
View File

@@ -191,7 +191,7 @@ public class ESTemplate {
StringBuilder sb = new StringBuilder();
esFieldData.forEach((key, value) -> {
if (value instanceof Map) {
HashMap mapValue = (HashMap) value;
Map<?, ?> mapValue = (Map<?, ?>) value;
if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) {
sb.append("ctx._source")
.append("['")

13
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java
View File

@@ -1,6 +1,7 @@
package com.alibaba.otter.canal.client.adapter.hbase.service;
import java.sql.*;
import java.sql.ResultSetMetaData;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -10,19 +11,23 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import javax.sql.DataSource;
import com.alibaba.otter.canal.client.adapter.support.Util;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig;
import com.alibaba.otter.canal.client.adapter.hbase.support.*;
import com.alibaba.otter.canal.client.adapter.hbase.support.HRow;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhType;
import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil;
import com.alibaba.otter.canal.client.adapter.hbase.support.Type;
import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.google.common.base.Joiner;
/**

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java
View File

@@ -151,7 +151,7 @@ public class CanalAdapterLoader {
evnProperties = new Properties();
for (PropertySource<?> propertySource : ((StandardEnvironment) env).getPropertySources()) {
if (propertySource instanceof EnumerablePropertySource) {
String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames();
String[] names = ((EnumerablePropertySource<?>) propertySource).getPropertyNames();
for (String name : names) {
Object val = propertySource.getProperty(name);
if (val != null) {

2
client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java
View File

@@ -139,9 +139,7 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader {
/**
* 加载有变动的adapter配置
*/
@SuppressWarnings("unchecked")
private void loadModifiedAdapterConfigs() {
Map<String, ConfigItem>[] res = new Map[2];
Map<String, ConfigItem> remoteConfigStatus = new HashMap<>();
String sql = "select id, category, name, modified_time from canal_adapter_config";
try (Connection conn = dataSource.getConnection();

4
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java
View File

@@ -4,12 +4,10 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
/**

1
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java
View File

@@ -4,7 +4,6 @@ import java.io.File;
import java.util.HashMap;
import java.util.Map;
import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
import org.apache.commons.io.monitor.FileAlterationMonitor;

15
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java
View File

@@ -3,8 +3,16 @@ package com.alibaba.otter.canal.client.adapter.rdb.service;
import java.sql.Connection;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.*;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Function;
import javax.sql.DataSource;
@@ -51,7 +59,6 @@ public class RdbSyncService {
return columnsTypeCache;
}
@SuppressWarnings("unchecked")
public RdbSyncService(DataSource dataSource, Integer threads, boolean skipDupException){
this(dataSource, threads, new ConcurrentHashMap<>(), skipDupException);
}
@@ -95,7 +102,7 @@ public class RdbSyncService {
}
}
if (toExecute) {
List<Future> futures = new ArrayList<>();
List<Future<Boolean>> futures = new ArrayList<>();
for (int i = 0; i < threads; i++) {
int j = i;
futures.add(executorThreads[i].submit(() -> {

1
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
View File

@@ -4,7 +4,6 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.MapMaker;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

5
client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java
View File

@@ -11,12 +11,15 @@ import org.springframework.beans.BeanUtils;
* @Date 2018/12/17
*/
public class KafkaFlatMessage extends FlatMessage {
private static final long serialVersionUID = 5748024400508080710L;
/**
* Kafka 消息 offset
*/
private long offset;
public KafkaFlatMessage(FlatMessage message, long offset) {
public KafkaFlatMessage(FlatMessage message, long offset){
super(message.getId());
BeanUtils.copyProperties(message, this);
this.offset = offset;

6
client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java
View File

@@ -11,18 +11,20 @@ import org.springframework.beans.BeanUtils;
* @Date 2018/12/17
*/
public class KafkaMessage extends Message {
private static final long serialVersionUID = -293120358490119447L;
/**
* Kafka 消息 offset
*/
private long offset;
public KafkaMessage(Message message, long offset) {
public KafkaMessage(Message message, long offset){
super(message.getId());
BeanUtils.copyProperties(message, this);
this.offset = offset;
}
public long getOffset() {
return offset;
}