chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

= code cleanup

Browse Source 
- for loop replaceable with enhanced 'for' loop
  - using diamond in generic type
  - collapse try catch exception
  - using try-with-resource
  - using  List#sort instead of Collections#sort
  - replaced lambda with method reference
  - using native compare method to compare numbers
  - using switch instead of if for in ESSyncUtil
This commit is contained in:
zavakid
2020-09-17 22:49:35 +08:00
parent 43c23ffb97
commit df24596092
33 changed files with 268 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java
View File

@@ -70,8 +70,8 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
return false;
}
}
for (int i = 0; i < match.length; i++) {
if (match[i]) {
for (boolean b : match) {
if (b) {
return true;
}
}

7
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java
View File

@@ -130,8 +130,7 @@ public abstract class YamlProcessor {
if (logger.isDebugEnabled()) {
logger.debug("Loading from YAML: " + resource);
}
Reader reader = new UnicodeReader(resource.getInputStream());
try {
try (Reader reader = new UnicodeReader(resource.getInputStream())) {
for (Object object : yaml.loadAll(reader)) {
if (object != null && process(asMap(object), callback)) {
count++;
@@ -142,10 +141,8 @@ public abstract class YamlProcessor {
}
if (logger.isDebugEnabled()) {
logger.debug(
"Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
"Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource);
}
} finally {
reader.close();
}
} catch (IOException ex) {
handleProcessError(resource, ex);

37
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java
View File

@@ -6,6 +6,7 @@ import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
@@ -317,10 +318,8 @@ public class ExtensionLoader<T> {
while (urls.hasMoreElements()) {
URL url = urls.nextElement();
try {
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(url.openStream(), "utf-8"));
String line = null;
try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
final int ci = line.indexOf('#');
if (ci >= 0) line = line.substring(0, ci);
@@ -340,10 +339,10 @@ public class ExtensionLoader<T> {
// classLoader);
if (!type.isAssignableFrom(clazz)) {
throw new IllegalStateException(
"Error when load extension class(interface: " + type
+ ", class line: " + clazz.getName()
+ "), class " + clazz.getName()
+ "is not subtype of interface.");
"Error when load extension class(interface: " + type
+ ", class line: " + clazz.getName()
+ "), class " + clazz.getName()
+ "is not subtype of interface.");
} else {
try {
clazz.getConstructor(type);
@@ -361,10 +360,10 @@ public class ExtensionLoader<T> {
} else if (c != clazz) {
cachedNames.remove(clazz);
throw new IllegalStateException(
"Duplicate extension " + type.getName() + " name "
+ n + " on "
+ c.getName() + " and "
+ clazz.getName());
"Duplicate extension " + type.getName() + " name "
+ n + " on "
+ c.getName() + " and "
+ clazz.getName());
}
}
}
@@ -373,19 +372,15 @@ public class ExtensionLoader<T> {
}
} catch (Throwable t) {
IllegalStateException e = new IllegalStateException(
"Failed to load extension class(interface: " + type + ", class line: "
+ line + ") in " + url
+ ", cause: "
+ t.getMessage(),
t);
"Failed to load extension class(interface: " + type + ", class line: "
+ line + ") in " + url
+ ", cause: "
+ t.getMessage(),
t);
exceptions.put(line, e);
}
}
} // end of while read lines
} finally {
if (reader != null) {
reader.close();
}
}
} catch (Throwable t) {
logger.error("Exception when load extension class(interface: " + type + ", class file: " + url

303
client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java
View File

@@ -63,172 +63,187 @@ public class ESSyncUtil {
return val;
}
Object res = null;
if ("integer".equals(esType)) {
if (val instanceof Number) {
res = ((Number) val).intValue();
} else {
res = Integer.parseInt(val.toString());
}
} else if ("long".equals(esType)) {
if (val instanceof Number) {
res = ((Number) val).longValue();
} else {
res = Long.parseLong(val.toString());
}
} else if ("short".equals(esType)) {
if (val instanceof Number) {
res = ((Number) val).shortValue();
} else {
res = Short.parseShort(val.toString());
}
} else if ("byte".equals(esType)) {
if (val instanceof Number) {
res = ((Number) val).byteValue();
} else {
res = Byte.parseByte(val.toString());
}
} else if ("double".equals(esType)) {
if (val instanceof Number) {
res = ((Number) val).doubleValue();
} else {
res = Double.parseDouble(val.toString());
}
} else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) {
if (val instanceof Number) {
res = ((Number) val).floatValue();
} else {
res = Float.parseFloat(val.toString());
}
} else if ("boolean".equals(esType)) {
if (val instanceof Boolean) {
res = val;
} else if (val instanceof Number) {
int v = ((Number) val).intValue();
res = v != 0;
} else {
res = Boolean.parseBoolean(val.toString());
}
} else if ("date".equals(esType)) {
if (val instanceof java.sql.Time) {
DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("HH:mm:ss.SSS");
switch (esType) {
case "integer":
if (val instanceof Number) {
res = ((Number) val).intValue();
} else {
res = dateTime.toString("HH:mm:ss");
res = Integer.parseInt(val.toString());
}
} else if (val instanceof java.sql.Timestamp) {
DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
break;
case "long":
if (val instanceof Number) {
res = ((Number) val).longValue();
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
res = Long.parseLong(val.toString());
}
} else if (val instanceof java.sql.Date || val instanceof Date) {
DateTime dateTime;
if (val instanceof java.sql.Date) {
dateTime = new DateTime(((java.sql.Date) val).getTime());
break;
case "short":
if (val instanceof Number) {
res = ((Number) val).shortValue();
} else {
dateTime = new DateTime(((Date) val).getTime());
res = Short.parseShort(val.toString());
}
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
&& dateTime.getMillisOfSecond() == 0) {
res = dateTime.toString("yyyy-MM-dd");
break;
case "byte":
if (val instanceof Number) {
res = ((Number) val).byteValue();
} else {
res = Byte.parseByte(val.toString());
}
break;
case "double":
if (val instanceof Number) {
res = ((Number) val).doubleValue();
} else {
res = Double.parseDouble(val.toString());
}
break;
case "float":
case "half_float":
case "scaled_float":
if (val instanceof Number) {
res = ((Number) val).floatValue();
} else {
res = Float.parseFloat(val.toString());
}
break;
case "boolean":
if (val instanceof Boolean) {
res = val;
} else if (val instanceof Number) {
int v = ((Number) val).intValue();
res = v != 0;
} else {
res = Boolean.parseBoolean(val.toString());
}
break;
case "date":
if (val instanceof java.sql.Time) {
DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("HH:mm:ss.SSS");
} else {
res = dateTime.toString("HH:mm:ss");
}
} else if (val instanceof java.sql.Timestamp) {
DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
}
}
} else if (val instanceof Long) {
DateTime dateTime = new DateTime(((Long) val).longValue());
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
&& dateTime.getMillisOfSecond() == 0) {
res = dateTime.toString("yyyy-MM-dd");
} else if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
}
} else if (val instanceof String) {
String v = ((String) val).trim();
if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
&& v.charAt(13) == ':' && v.charAt(16) == ':') {
String dt = v.substring(0, 10) + "T" + v.substring(11);
Date date = Util.parseDate(dt);
if (date != null) {
DateTime dateTime = new DateTime(date);
} else if (val instanceof java.sql.Date || val instanceof Date) {
DateTime dateTime;
if (val instanceof java.sql.Date) {
dateTime = new DateTime(((java.sql.Date) val).getTime());
} else {
dateTime = new DateTime(((Date) val).getTime());
}
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
&& dateTime.getMillisOfSecond() == 0) {
res = dateTime.toString("yyyy-MM-dd");
} else {
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
}
}
} else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
Date date = Util.parseDate(v);
if (date != null) {
DateTime dateTime = new DateTime(date);
} else if (val instanceof Long) {
DateTime dateTime = new DateTime(((Long) val).longValue());
if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
&& dateTime.getMillisOfSecond() == 0) {
res = dateTime.toString("yyyy-MM-dd");
} else if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
}
} else if (val instanceof String) {
String v = ((String) val).trim();
if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
&& v.charAt(13) == ':' && v.charAt(16) == ':') {
String dt = v.substring(0, 10) + "T" + v.substring(11);
Date date = Util.parseDate(dt);
if (date != null) {
DateTime dateTime = new DateTime(date);
if (dateTime.getMillisOfSecond() != 0) {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
} else {
res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
}
}
} else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
Date date = Util.parseDate(v);
if (date != null) {
DateTime dateTime = new DateTime(date);
res = dateTime.toString("yyyy-MM-dd");
}
}
}
}
} else if ("binary".equals(esType)) {
if (val instanceof byte[]) {
Base64 base64 = new Base64();
res = base64.encodeAsString((byte[]) val);
} else if (val instanceof Blob) {
byte[] b = blobToBytes((Blob) val);
Base64 base64 = new Base64();
res = base64.encodeAsString(b);
} else if (val instanceof String) {
// 对应canal中的单字节编码
byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
Base64 base64 = new Base64();
res = base64.encodeAsString(b);
}
} else if ("geo_point".equals(esType)) {
if (!(val instanceof String)) {
logger.error("es type is geo_point, but source type is not String");
return val;
}
if (!((String) val).contains(",")) {
logger.error("es type is geo_point, source value not contains ',' separator");
return val;
}
String[] point = ((String) val).split(",");
Map<String, Double> location = new HashMap<>();
location.put("lat", Double.valueOf(point[0].trim()));
location.put("lon", Double.valueOf(point[1].trim()));
return location;
} else if ("array".equals(esType)) {
if ("".equals(val.toString().trim())) {
res = new ArrayList<>();
} else {
String value = val.toString();
String separator = ",";
if (!value.contains(",")) {
if (value.contains(";")) {
separator = ";";
} else if (value.contains("|")) {
separator = "\\|";
} else if (value.contains("-")) {
separator = "-";
}
break;
case "binary":
if (val instanceof byte[]) {
Base64 base64 = new Base64();
res = base64.encodeAsString((byte[]) val);
} else if (val instanceof Blob) {
byte[] b = blobToBytes((Blob) val);
Base64 base64 = new Base64();
res = base64.encodeAsString(b);
} else if (val instanceof String) {
// 对应canal中的单字节编码
byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
Base64 base64 = new Base64();
res = base64.encodeAsString(b);
}
String[] values = value.split(separator);
return Arrays.asList(values);
}
} else if ("object".equals(esType)) {
if ("".equals(val.toString().trim())) {
res = new HashMap<>();
} else {
res = JSON.parseObject(val.toString(), Map.class);
}
} else {
// 其他类全以字符串处理
res = val.toString();
break;
case "geo_point":
if (!(val instanceof String)) {
logger.error("es type is geo_point, but source type is not String");
return val;
}
if (!((String) val).contains(",")) {
logger.error("es type is geo_point, source value not contains ',' separator");
return val;
}
String[] point = ((String) val).split(",");
Map<String, Double> location = new HashMap<>();
location.put("lat", Double.valueOf(point[0].trim()));
location.put("lon", Double.valueOf(point[1].trim()));
return location;
case "array":
if ("".equals(val.toString().trim())) {
res = new ArrayList<>();
} else {
String value = val.toString();
String separator = ",";
if (!value.contains(",")) {
if (value.contains(";")) {
separator = ";";
} else if (value.contains("|")) {
separator = "\\|";
} else if (value.contains("-")) {
separator = "-";
}
}
String[] values = value.split(separator);
return Arrays.asList(values);
}
break;
case "object":
if ("".equals(val.toString().trim())) {
res = new HashMap<>();
} else {
res = JSON.parseObject(val.toString(), Map.class);
}
break;
default:
// 其他类全以字符串处理
res = val.toString();
break;
}
return res;

71
client-adapter/kudu/src/main/java/com/alibaba/otter/canal/client/adapter/kudu/service/KuduEtlService.java
View File

@@ -88,45 +88,40 @@ public class KuduEtlService extends AbstractEtlService {
Map<String, Object> data = new HashMap<>();
for (Map.Entry<String, String> entry : columnsMap.entrySet()) {
String mysqlColumnName = entry.getKey();// mysql字段名
String kuduColumnName = entry.getValue();// kudu字段名
if (kuduColumnName == null) {
kuduColumnName = mysqlColumnName;
}
Object value = rs.getObject(kuduColumnName);
if (value != null) {
data.put(kuduColumnName, value);
} else {
data.put(kuduColumnName, null);
}
}
dataList.add(data);
idx++;
impCount.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug("successful import count:" + impCount.get());
}
if (idx % kuduMapping.getCommitBatch() == 0) {
kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
dataList.clear();
}
}
if (!dataList.isEmpty()) {
kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
}
return true;
String kuduColumnName = entry.getValue();// kudu字段名
if (kuduColumnName == null) {
kuduColumnName = mysqlColumnName;
}
Object value = rs.getObject(kuduColumnName);
if (value != null) {
data.put(kuduColumnName, value);
} else {
data.put(kuduColumnName, null);
}
}
dataList.add(data);
idx++;
impCount.incrementAndGet();
if (logger.isDebugEnabled()) {
logger.debug("successful import count:" + impCount.get());
}
if (idx % kuduMapping.getCommitBatch() == 0) {
kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
dataList.clear();
}
}
if (!dataList.isEmpty()) {
kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
}
return true;
} catch (SQLException e) {
e.printStackTrace();
logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
return false;
} catch (KuduException e) {
e.printStackTrace();
logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
return false;
}
} );
} catch (SQLException | KuduException e) {
e.printStackTrace();
logger.error(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage(), e);
errMsg.add(kuduMapping.getTargetTable() + " etl failed! ==>" + e.getMessage());
return false;
}
});
return true;
} catch (Exception e) {
logger.error(e.getMessage(), e);