From f21abc13af4fa9fa843f1c4e16d3128948d3c605 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 26 Feb 2019 10:23:38 +0800 Subject: [PATCH 1/8] fixed #1538 --- .../client/adapter/support/JdbcTypeUtil.java | 7 ++++--- .../client/adapter/es/support/ESSyncUtil.java | 17 +++++++++-------- .../adapter/hbase/support/PhTypeUtil.java | 7 ++++--- .../client/adapter/rdb/support/SyncUtil.java | 9 +++++---- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java index c24ff7de..6843665b 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java @@ -5,6 +5,7 @@ import java.math.BigInteger; import java.sql.*; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,7 +117,7 @@ public class JdbcTypeUtil { case Types.DATE: if (!value.startsWith("0000-00-00")) { value = value.trim().replace(" ", "T"); - DateTime dt = new DateTime(value); + DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00")); res = new Date(dt.toDate().getTime()); } else { res = null; @@ -124,13 +125,13 @@ public class JdbcTypeUtil { break; case Types.TIME: value = "T" + value; - DateTime dt = new DateTime(value); + DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00")); res = new Time(dt.toDate().getTime()); break; case Types.TIMESTAMP: if (!value.startsWith("0000-00-00")) { value = value.trim().replace(" ", "T"); - dt = new DateTime(value); + dt = new DateTime(value, DateTimeZone.forID("+08:00")); res = new Timestamp(dt.toDate().getTime()); } else { res = null; diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index cf8f8fd4..74ca70cd 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -12,6 +12,7 @@ import javax.sql.DataSource; import org.apache.commons.codec.binary.Base64; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,14 +104,14 @@ public class ESSyncUtil { } } else if ("date".equals(esType)) { if (val instanceof java.sql.Time) { - DateTime dateTime = new DateTime(((java.sql.Time) val).getTime()); + DateTime dateTime = new DateTime(((java.sql.Time) val).getTime(), DateTimeZone.forID("+08:00")); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("HH:mm:ss.SSS"); } else { res = dateTime.toString("HH:mm:ss"); } } else if (val instanceof java.sql.Timestamp) { - DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime()); + DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime(), DateTimeZone.forID("+08:00")); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); } else { @@ -119,9 +120,9 @@ public class ESSyncUtil { } else if (val instanceof java.sql.Date || val instanceof Date) { DateTime dateTime; if (val instanceof java.sql.Date) { - dateTime = new DateTime(((java.sql.Date) val).getTime()); + dateTime = new DateTime(((java.sql.Date) val).getTime(), DateTimeZone.forID("+08:00")); } else { - dateTime = new DateTime(((Date) val).getTime()); + dateTime = new DateTime(((Date) val).getTime(), DateTimeZone.forID("+08:00")); } if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 && dateTime.getMillisOfSecond() == 0) { @@ -134,7 +135,7 @@ public class ESSyncUtil { } } } else if (val instanceof Long) { - DateTime dateTime = new DateTime(((Long) val).longValue()); + DateTime dateTime = new DateTime(((Long) val).longValue(), DateTimeZone.forID("+08:00")); if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 && dateTime.getMillisOfSecond() == 0) { res = dateTime.toString("yyyy-MM-dd"); @@ -148,14 +149,14 @@ public class ESSyncUtil { if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' ' && v.charAt(13) == ':' && v.charAt(16) == ':') { String dt = v.substring(0, 10) + "T" + v.substring(11); - DateTime dateTime = new DateTime(dt); + DateTime dateTime = new DateTime(dt, DateTimeZone.forID("+08:00")); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); } else { res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); } } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') { - DateTime dateTime = new DateTime(v); + DateTime dateTime = new DateTime(v, DateTimeZone.forID("+08:00")); res = dateTime.toString("yyyy-MM-dd"); } } @@ -237,7 +238,7 @@ public class ESSyncUtil { /** * 拼接主键条件 - * + * * @param mapping * @param data * @return diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java index a3a8e0b9..f738e054 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java @@ -11,6 +11,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.joda.time.DateTime; import com.google.common.math.LongMath; +import org.joda.time.DateTimeZone; /** * Phoenix类型转换工具类 @@ -608,12 +609,12 @@ public class PhTypeUtil { Date date = null; int len = dateStr.length(); if (len == 10 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-') { - date = new DateTime(dateStr).toDate(); + date = new DateTime(dateStr, DateTimeZone.forID("+08:00")).toDate(); } else if (len == 8 && dateStr.charAt(2) == ':' && dateStr.charAt(5) == ':') { - date = new DateTime("T" + dateStr).toDate(); + date = new DateTime("T" + dateStr, DateTimeZone.forID("+08:00")).toDate(); } else if (len >= 19 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-' && dateStr.charAt(13) == ':' && dateStr.charAt(16) == ':') { - date = new DateTime(dateStr.replace(" ", "T")).toDate(); + date = new DateTime(dateStr.replace(" ", "T"), DateTimeZone.forID("+08:00")).toDate(); } return date; } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java index 6be006d2..ea1977e9 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java @@ -13,6 +13,7 @@ import org.apache.commons.lang.StringUtils; import org.joda.time.DateTime; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import org.joda.time.DateTimeZone; public class SyncUtil { @@ -86,7 +87,7 @@ public class SyncUtil { } break; case Types.TINYINT: - if (value instanceof Number) { + if (value instanceof Number) { pstmt.setByte(i, ((Number) value).byteValue()); } else if (value instanceof String) { pstmt.setByte(i, Byte.parseByte((String) value)); @@ -197,7 +198,7 @@ public class SyncUtil { String v = (String) value; if (!v.startsWith("0000-00-00")) { v = v.trim().replace(" ", "T"); - DateTime dt = new DateTime(v); + DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); pstmt.setDate(i, new Date(dt.toDate().getTime())); } else { pstmt.setNull(i, type); @@ -214,7 +215,7 @@ public class SyncUtil { } else if (value instanceof String) { String v = (String) value; v = "T" + v; - DateTime dt = new DateTime(v); + DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); pstmt.setTime(i, new Time(dt.toDate().getTime())); } else { pstmt.setNull(i, type); @@ -229,7 +230,7 @@ public class SyncUtil { String v = (String) value; if (!v.startsWith("0000-00-00")) { v = v.trim().replace(" ", "T"); - DateTime dt = new DateTime(v); + DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); pstmt.setTimestamp(i, new Timestamp(dt.toDate().getTime())); } else { pstmt.setNull(i, type); From e7f069850f74b27b5a481bf56b711b6d3cdad330 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 28 Feb 2019 17:29:06 +0800 Subject: [PATCH 2/8] fixed #1538 --- .../client/adapter/support/JdbcTypeUtil.java | 32 ++++++++++------- .../canal/client/adapter/support/Util.java | 35 +++++++++++++++++++ .../client/adapter/es/support/ESSyncUtil.java | 32 ++++++++++------- .../adapter/hbase/support/PhTypeUtil.java | 22 +++--------- .../client/adapter/rdb/support/SyncUtil.java | 30 ++++++++++------ 5 files changed, 98 insertions(+), 53 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java index 6843665b..cc1e4e7e 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtil.java @@ -4,8 +4,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.sql.*; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,23 +114,33 @@ public class JdbcTypeUtil { break; case Types.DATE: if (!value.startsWith("0000-00-00")) { - value = value.trim().replace(" ", "T"); - DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00")); - res = new Date(dt.toDate().getTime()); + java.util.Date date = Util.parseDate(value); + if (date != null) { + res = new Date(date.getTime()); + } else { + res = null; + } } else { res = null; } break; - case Types.TIME: - value = "T" + value; - DateTime dt = new DateTime(value, DateTimeZone.forID("+08:00")); - res = new Time(dt.toDate().getTime()); + case Types.TIME: { + java.util.Date date = Util.parseDate(value); + if (date != null) { + res = new Time(date.getTime()); + } else { + res = null; + } break; + } case Types.TIMESTAMP: if (!value.startsWith("0000-00-00")) { - value = value.trim().replace(" ", "T"); - dt = new DateTime(value, DateTimeZone.forID("+08:00")); - res = new Timestamp(dt.toDate().getTime()); + java.util.Date date = Util.parseDate(value); + if (date != null) { + res = new Timestamp(date.getTime()); + } else { + res = null; + } } else { res = null; } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java index 919974a1..42cc2ca1 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -6,12 +6,15 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Date; import java.util.function.Consumer; import java.util.function.Function; import javax.sql.DataSource; import org.apache.commons.lang.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -136,4 +139,36 @@ public class Util { return column; } + + /** + * 通用日期时间字符解析 + * + * @param datetimeStr 日期时间字符串 + * @return Date + */ + public static Date parseDate(String datetimeStr) { + if (StringUtils.isEmpty(datetimeStr)) { + return null; + } + datetimeStr = datetimeStr.trim(); + if (datetimeStr.contains("-")) { + if (datetimeStr.contains(":")) { + datetimeStr = datetimeStr.replace(" ", "T"); + } + } else if (datetimeStr.contains(":")) { + datetimeStr = "T" + datetimeStr; + } + + DateTime dateTime; + if (datetimeStr.startsWith("1940-06-03") || datetimeStr.startsWith("1941-03-16") + || datetimeStr.startsWith("1986-05-04") || datetimeStr.startsWith("1987-04-12") + || datetimeStr.startsWith("1988-04-10") || datetimeStr.startsWith("1989-04-16") + || datetimeStr.startsWith("1990-04-15") || datetimeStr.startsWith("1991-04-14")) { + dateTime = new DateTime(datetimeStr, DateTimeZone.forID("+08:00")); + } else { + dateTime = new DateTime(datetimeStr); + } + + return dateTime.toDate(); + } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java index 74ca70cd..84cb2f78 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java @@ -12,7 +12,6 @@ import javax.sql.DataSource; import org.apache.commons.codec.binary.Base64; import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,6 +20,7 @@ import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem; +import com.alibaba.otter.canal.client.adapter.support.Util; /** * ES 同步工具同类 @@ -104,14 +104,14 @@ public class ESSyncUtil { } } else if ("date".equals(esType)) { if (val instanceof java.sql.Time) { - DateTime dateTime = new DateTime(((java.sql.Time) val).getTime(), DateTimeZone.forID("+08:00")); + DateTime dateTime = new DateTime(((java.sql.Time) val).getTime()); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("HH:mm:ss.SSS"); } else { res = dateTime.toString("HH:mm:ss"); } } else if (val instanceof java.sql.Timestamp) { - DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime(), DateTimeZone.forID("+08:00")); + DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime()); if (dateTime.getMillisOfSecond() != 0) { res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); } else { @@ -120,9 +120,9 @@ public class ESSyncUtil { } else if (val instanceof java.sql.Date || val instanceof Date) { DateTime dateTime; if (val instanceof java.sql.Date) { - dateTime = new DateTime(((java.sql.Date) val).getTime(), DateTimeZone.forID("+08:00")); + dateTime = new DateTime(((java.sql.Date) val).getTime()); } else { - dateTime = new DateTime(((Date) val).getTime(), DateTimeZone.forID("+08:00")); + dateTime = new DateTime(((Date) val).getTime()); } if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 && dateTime.getMillisOfSecond() == 0) { @@ -135,7 +135,7 @@ public class ESSyncUtil { } } } else if (val instanceof Long) { - DateTime dateTime = new DateTime(((Long) val).longValue(), DateTimeZone.forID("+08:00")); + DateTime dateTime = new DateTime(((Long) val).longValue()); if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0 && dateTime.getMillisOfSecond() == 0) { res = dateTime.toString("yyyy-MM-dd"); @@ -149,15 +149,21 @@ public class ESSyncUtil { if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' ' && v.charAt(13) == ':' && v.charAt(16) == ':') { String dt = v.substring(0, 10) + "T" + v.substring(11); - DateTime dateTime = new DateTime(dt, DateTimeZone.forID("+08:00")); - if (dateTime.getMillisOfSecond() != 0) { - res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); - } else { - res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + Date date = Util.parseDate(dt); + if (date != null) { + DateTime dateTime = new DateTime(date); + if (dateTime.getMillisOfSecond() != 0) { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS+08:00"); + } else { + res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss+08:00"); + } } } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') { - DateTime dateTime = new DateTime(v, DateTimeZone.forID("+08:00")); - res = dateTime.toString("yyyy-MM-dd"); + Date date = Util.parseDate(v); + if (date != null) { + DateTime dateTime = new DateTime(date); + res = dateTime.toString("yyyy-MM-dd"); + } } } } else if ("binary".equals(esType)) { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java index f738e054..6ff684e4 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/support/PhTypeUtil.java @@ -7,6 +7,7 @@ import java.math.RoundingMode; import java.sql.Timestamp; import java.util.Date; +import com.alibaba.otter.canal.client.adapter.support.Util; import org.apache.hadoop.hbase.util.Bytes; import org.joda.time.DateTime; @@ -390,7 +391,7 @@ public class PhTypeUtil { String dateStr = (String) v; Date date; try { - date = parseDatetime(dateStr); + date = Util.parseDate(dateStr); if (date != null) { encodeLong(date.getTime(), b, 0); } @@ -419,7 +420,7 @@ public class PhTypeUtil { String dateStr = (String) v; Date date; try { - date = parseDatetime(dateStr); + date = Util.parseDate(dateStr); if (date != null) { encodeUnsignedLong(date.getTime(), b, 0); } @@ -600,22 +601,9 @@ public class PhTypeUtil { private static void checkForSufficientLength(byte[] b, int offset, int requiredLength) { if (b.length < offset + requiredLength) { - throw new RuntimeException("Expected length of at least " + requiredLength + " bytes, but had " - + (b.length - offset)); + throw new RuntimeException( + "Expected length of at least " + requiredLength + " bytes, but had " + (b.length - offset)); } } - private static Date parseDatetime(String dateStr) { - Date date = null; - int len = dateStr.length(); - if (len == 10 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-') { - date = new DateTime(dateStr, DateTimeZone.forID("+08:00")).toDate(); - } else if (len == 8 && dateStr.charAt(2) == ':' && dateStr.charAt(5) == ':') { - date = new DateTime("T" + dateStr, DateTimeZone.forID("+08:00")).toDate(); - } else if (len >= 19 && dateStr.charAt(4) == '-' && dateStr.charAt(7) == '-' && dateStr.charAt(13) == ':' - && dateStr.charAt(16) == ':') { - date = new DateTime(dateStr.replace(" ", "T"), DateTimeZone.forID("+08:00")).toDate(); - } - return date; - } } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java index ea1977e9..a8670766 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/support/SyncUtil.java @@ -10,10 +10,9 @@ import java.util.LinkedHashMap; import java.util.Map; import org.apache.commons.lang.StringUtils; -import org.joda.time.DateTime; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; -import org.joda.time.DateTimeZone; +import com.alibaba.otter.canal.client.adapter.support.Util; public class SyncUtil { @@ -197,9 +196,12 @@ public class SyncUtil { } else if (value instanceof String) { String v = (String) value; if (!v.startsWith("0000-00-00")) { - v = v.trim().replace(" ", "T"); - DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); - pstmt.setDate(i, new Date(dt.toDate().getTime())); + java.util.Date date = Util.parseDate(v); + if (date != null) { + pstmt.setDate(i, new Date(date.getTime())); + } else { + pstmt.setNull(i, type); + } } else { pstmt.setNull(i, type); } @@ -214,9 +216,12 @@ public class SyncUtil { pstmt.setTime(i, new java.sql.Time(((java.util.Date) value).getTime())); } else if (value instanceof String) { String v = (String) value; - v = "T" + v; - DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); - pstmt.setTime(i, new Time(dt.toDate().getTime())); + java.util.Date date = Util.parseDate(v); + if (date != null) { + pstmt.setTime(i, new Time(date.getTime())); + } else { + pstmt.setNull(i, type); + } } else { pstmt.setNull(i, type); } @@ -229,9 +234,12 @@ public class SyncUtil { } else if (value instanceof String) { String v = (String) value; if (!v.startsWith("0000-00-00")) { - v = v.trim().replace(" ", "T"); - DateTime dt = new DateTime(v, DateTimeZone.forID("+08:00")); - pstmt.setTimestamp(i, new Timestamp(dt.toDate().getTime())); + java.util.Date date = Util.parseDate(v); + if (date != null) { + pstmt.setTimestamp(i, new Timestamp(date.getTime())); + } else { + pstmt.setNull(i, type); + } } else { pstmt.setNull(i, type); } From 8187d7c5df34c3f6b3714d1439d733a3957777f4 Mon Sep 17 00:00:00 2001 From: lee123lee123 Date: Thu, 28 Feb 2019 19:04:14 +0800 Subject: [PATCH 3/8] =?UTF-8?q?Issues=20#1460=20=20=E4=BF=AE=E6=94=B9utf8?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=B8=A6=E8=A1=A8=E6=83=85=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/client/adapter/es/test/sync/db_schema.sql | 6 +++--- .../src/main/resources/spring/tsdb/sql/create_table.sql | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql index bff2e118..c7274ca2 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql @@ -7,7 +7,7 @@ CREATE TABLE `label` ( `user_id` bigint(20) NOT NULL, `label` varchar(30) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4; -- ---------------------------- -- Table structure for role @@ -17,7 +17,7 @@ CREATE TABLE `role` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `role_name` varchar(30) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4; -- ---------------------------- -- Table structure for user @@ -29,7 +29,7 @@ CREATE TABLE `user` ( `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, `role_id` bigint(20) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4; insert into user (id,name,role_id) values (1,'Eric',1); insert into role (id,role_name) values (1,'admin'); diff --git a/deployer/src/main/resources/spring/tsdb/sql/create_table.sql b/deployer/src/main/resources/spring/tsdb/sql/create_table.sql index 322bc2b6..8cb82c82 100644 --- a/deployer/src/main/resources/spring/tsdb/sql/create_table.sql +++ b/deployer/src/main/resources/spring/tsdb/sql/create_table.sql @@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS `meta_snapshot` ( KEY `destination` (`destination`), KEY `destination_timestamp` (`destination`,`binlog_timestamp`), KEY `gmt_modified` (`gmt_modified`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表'; +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构记录表快照表'; CREATE TABLE IF NOT EXISTS `meta_history` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -36,4 +36,4 @@ CREATE TABLE IF NOT EXISTS `meta_history` ( KEY `destination` (`destination`), KEY `destination_timestamp` (`destination`,`binlog_timestamp`), KEY `gmt_modified` (`gmt_modified`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表'; \ No newline at end of file +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='表结构变化明细表'; \ No newline at end of file From b891afa50eab722cc3ba7fa8f4e1b63f7c38e60b Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 28 Feb 2019 19:28:17 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=B6=E5=8C=BA?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/support/Util.java | 112 ++++++++++++++++-- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java index 42cc2ca1..ff97e4f7 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -6,7 +6,10 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.time.*; +import java.time.format.DateTimeFormatter; import java.util.Date; +import java.util.TimeZone; import java.util.function.Consumer; import java.util.function.Function; @@ -18,6 +21,10 @@ import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + public class Util { private static final Logger logger = LoggerFactory.getLogger(Util.class); @@ -140,6 +147,17 @@ public class Util { return column; } + private static String UTC; // 当前时区 + + static { + TimeZone timeZone = TimeZone.getTimeZone(ZoneId.systemDefault().getId()); + int currentTimeZone = timeZone.getOffset(System.currentTimeMillis()) * 10 / (1000 * 3600); + int tmp = currentTimeZone % 10; + int m = 60 * tmp / 10; + UTC = "+" + String.format("%02d", currentTimeZone / 10) + ":" + String.format("%02d", m); + TimeZone.setDefault(TimeZone.getTimeZone("GMT" + UTC)); + } + /** * 通用日期时间字符解析 * @@ -159,16 +177,92 @@ public class Util { datetimeStr = "T" + datetimeStr; } - DateTime dateTime; - if (datetimeStr.startsWith("1940-06-03") || datetimeStr.startsWith("1941-03-16") - || datetimeStr.startsWith("1986-05-04") || datetimeStr.startsWith("1987-04-12") - || datetimeStr.startsWith("1988-04-10") || datetimeStr.startsWith("1989-04-16") - || datetimeStr.startsWith("1990-04-15") || datetimeStr.startsWith("1991-04-14")) { - dateTime = new DateTime(datetimeStr, DateTimeZone.forID("+08:00")); - } else { - dateTime = new DateTime(datetimeStr); - } + DateTime dateTime = new DateTime(datetimeStr, DateTimeZone.forID(UTC)); return dateTime.toDate(); } + + public static LoadingCache dateFormatterCache = CacheBuilder.newBuilder() + .build(new CacheLoader() { + + @Override + public DateTimeFormatter load(String key) { + return DateTimeFormatter.ofPattern(key); + } + }); + + public static Date parseDate2(String datetimeStr) { + if (StringUtils.isEmpty(datetimeStr)) { + return null; + } + try { + datetimeStr = datetimeStr.trim(); + int len = datetimeStr.length(); + if (datetimeStr.contains("-") && datetimeStr.contains(":") && datetimeStr.contains(".")) { + // 包含日期+时间+毫秒 + // 取毫秒位数 + int msLen = len - datetimeStr.indexOf(".") - 1; + StringBuilder ms = new StringBuilder(); + for (int i = 0; i < msLen; i++) { + ms.append("S"); + } + String formatter = "yyyy-MM-dd HH:mm:ss." + ms; + + DateTimeFormatter dateTimeFormatter = dateFormatterCache.get(formatter); + LocalDateTime dateTime = LocalDateTime.parse(datetimeStr, dateTimeFormatter); + return Date.from(dateTime.atZone(ZoneId.systemDefault()).toInstant()); + } else if (datetimeStr.contains("-") && datetimeStr.contains(":")) { + // 包含日期+时间 + // 判断包含时间位数 + int i = datetimeStr.indexOf(":"); + i = datetimeStr.indexOf(":", i + 1); + String formatter; + if (i > -1) { + formatter = "yyyy-MM-dd HH:mm:ss"; + } else { + formatter = "yyyy-MM-dd HH:mm"; + } + + DateTimeFormatter dateTimeFormatter = dateFormatterCache.get(formatter); + LocalDateTime dateTime = LocalDateTime.parse(datetimeStr, dateTimeFormatter); + return Date.from(dateTime.atZone(ZoneId.systemDefault()).toInstant()); + } else if (datetimeStr.contains("-")) { + // 只包含日期 + String formatter = "yyyy-MM-dd"; + DateTimeFormatter dateTimeFormatter = dateFormatterCache.get(formatter); + LocalDate localDate = LocalDate.parse(datetimeStr, dateTimeFormatter); + return Date.from(localDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()); + } else if (datetimeStr.contains(":")) { + // 只包含时间 + String formatter; + if (datetimeStr.contains(".")) { + // 包含毫秒 + int msLen = len - datetimeStr.indexOf(".") - 1; + StringBuilder ms = new StringBuilder(); + for (int i = 0; i < msLen; i++) { + ms.append("S"); + } + formatter = "HH:mm:ss." + ms; + } else { + // 判断包含时间位数 + int i = datetimeStr.indexOf(":"); + i = datetimeStr.indexOf(":", i + 1); + if (i > -1) { + formatter = "HH:mm:ss"; + } else { + formatter = "HH:mm"; + } + } + DateTimeFormatter dateTimeFormatter = dateFormatterCache.get(formatter); + LocalTime localTime = LocalTime.parse(datetimeStr, dateTimeFormatter); + LocalDate localDate = LocalDate.of(1970, Month.JANUARY, 1); + LocalDateTime localDateTime = LocalDateTime.of(localDate, localTime); + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + return null; + } } From 9888fae559a319e08e8a255725f0d711789d020c Mon Sep 17 00:00:00 2001 From: machey Date: Thu, 28 Feb 2019 23:22:02 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E8=B4=9F=E6=97=B6=E5=8C=BA=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/support/Util.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java index ff97e4f7..cca38080 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -2,10 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support; import java.io.File; import java.net.URL; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.time.*; import java.time.format.DateTimeFormatter; import java.util.Date; @@ -147,15 +144,22 @@ public class Util { return column; } - private static String UTC; // 当前时区 + private static String timeZone; // 当前时区 static { - TimeZone timeZone = TimeZone.getTimeZone(ZoneId.systemDefault().getId()); - int currentTimeZone = timeZone.getOffset(System.currentTimeMillis()) * 10 / (1000 * 3600); - int tmp = currentTimeZone % 10; - int m = 60 * tmp / 10; - UTC = "+" + String.format("%02d", currentTimeZone / 10) + ":" + String.format("%02d", m); - TimeZone.setDefault(TimeZone.getTimeZone("GMT" + UTC)); + TimeZone localTimeZone = TimeZone.getDefault(); + int rawOffset = localTimeZone.getRawOffset(); + String symbol = "+"; + if (rawOffset < 0) { + symbol = "-"; + } + rawOffset = Math.abs(rawOffset); + int offsetHore = rawOffset / 3600000; + int offsetMinute = rawOffset % 3600000 / 60000; + String hour = String.format("%1$02d", offsetHore); + String minute = String.format("%1$02d", offsetMinute); + timeZone = symbol + hour + ":" + minute; + TimeZone.setDefault(TimeZone.getTimeZone("GMT" + timeZone)); } /** @@ -177,12 +181,12 @@ public class Util { datetimeStr = "T" + datetimeStr; } - DateTime dateTime = new DateTime(datetimeStr, DateTimeZone.forID(UTC)); + DateTime dateTime = new DateTime(datetimeStr, DateTimeZone.forID(timeZone)); return dateTime.toDate(); } - public static LoadingCache dateFormatterCache = CacheBuilder.newBuilder() + private static LoadingCache dateFormatterCache = CacheBuilder.newBuilder() .build(new CacheLoader() { @Override From 5d4210bd221e3279a411d58ed8d2ef1c1363e8cc Mon Sep 17 00:00:00 2001 From: machey Date: Thu, 28 Feb 2019 23:31:07 +0800 Subject: [PATCH 6/8] modify --- .../com/alibaba/otter/canal/client/adapter/support/Util.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java index cca38080..54fb7fe4 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -154,9 +154,9 @@ public class Util { symbol = "-"; } rawOffset = Math.abs(rawOffset); - int offsetHore = rawOffset / 3600000; + int offsetHour = rawOffset / 3600000; int offsetMinute = rawOffset % 3600000 / 60000; - String hour = String.format("%1$02d", offsetHore); + String hour = String.format("%1$02d", offsetHour); String minute = String.format("%1$02d", offsetMinute); timeZone = symbol + hour + ":" + minute; TimeZone.setDefault(TimeZone.getTimeZone("GMT" + timeZone)); From 17813811877d870de8e09f152d68567c711f6a55 Mon Sep 17 00:00:00 2001 From: machey Date: Fri, 1 Mar 2019 00:19:47 +0800 Subject: [PATCH 7/8] modify --- .../com/alibaba/otter/canal/client/adapter/support/Util.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java index 54fb7fe4..f4878a47 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Util.java @@ -145,6 +145,7 @@ public class Util { } private static String timeZone; // 当前时区 + private static DateTimeZone dateTimeZone; static { TimeZone localTimeZone = TimeZone.getDefault(); @@ -159,6 +160,7 @@ public class Util { String hour = String.format("%1$02d", offsetHour); String minute = String.format("%1$02d", offsetMinute); timeZone = symbol + hour + ":" + minute; + dateTimeZone = DateTimeZone.forID(timeZone); TimeZone.setDefault(TimeZone.getTimeZone("GMT" + timeZone)); } @@ -181,7 +183,7 @@ public class Util { datetimeStr = "T" + datetimeStr; } - DateTime dateTime = new DateTime(datetimeStr, DateTimeZone.forID(timeZone)); + DateTime dateTime = new DateTime(datetimeStr, dateTimeZone); return dateTime.toDate(); } From 7abf50f233867934c9a4d0d5011a66cf3e1d782e Mon Sep 17 00:00:00 2001 From: dragonlong1986 Date: Sat, 2 Mar 2019 10:25:19 +0800 Subject: [PATCH 8/8] =?UTF-8?q?key=E6=94=B9=E6=88=90=E5=A4=A7=E5=86=99?= =?UTF-8?q?=EF=BC=8C=E5=B0=8F=E5=86=99=E4=B8=8D=E4=BC=9A=E7=94=9F=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deployer/src/main/resources/canal.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 8b5c09fc..30049c00 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -78,8 +78,8 @@ canal.instance.tsdb.snapshot.interval = 24 canal.instance.tsdb.snapshot.expire = 360 # aliyun ak/sk , support rds/mq -canal.aliyun.accesskey = -canal.aliyun.secretkey = +canal.aliyun.accessKey = +canal.aliyun.secretKey = ################################################# ######### destinations ############# @@ -117,4 +117,4 @@ canal.mq.compressionType = none canal.mq.acks = all # use transaction for kafka flatMessage batch produce canal.mq.transaction = false -#canal.mq.properties. = \ No newline at end of file +#canal.mq.properties. =