From 4bad0e88f849a9f7c16c012d7fec9bf2165ead05 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 21 Mar 2019 19:04:33 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20fix=20#1570?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 3 ++ .../client/adapter/es/test/SqlParseTest.java | 3 +- .../client/adapter/hbase/HbaseAdapter.java | 3 ++ .../src/main/resources/application.yml | 6 --- .../launcher/src/main/resources/logback.xml | 54 +++++++++++++++++++ .../adapter/logger/LoggerAdapterExample.java | 5 +- .../canal/client/adapter/rdb/RdbAdapter.java | 3 ++ 7 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 client-adapter/launcher/src/main/resources/logback.xml diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index cae10145..b41b857b 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -30,6 +30,7 @@ import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.support.*; +import org.slf4j.MDC; /** * ES外部适配器 @@ -70,6 +71,7 @@ public class ESAdapter implements OuterAdapter { @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { try { + MDC.put("adapter", "es"); this.envProperties = envProperties; Map esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); // 过滤不匹配的key的配置 @@ -239,6 +241,7 @@ public class ESAdapter implements OuterAdapter { if (transportClient != null) { transportClient.close(); } + MDC.remove("adapter"); } @Override diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java index 6c27e38b..648f6a0f 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java @@ -15,7 +15,8 @@ public class SqlParseTest { @Test public void parseTest() { - String sql = "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a " + String sql = "select a.id, CASE WHEN a.id <= 500 THEN '1' else '2' end as id2, " + + "concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a " + "left join role b on a.role_id=b.id " + "left join (select user_id, group_concat(label,',') as labels from user_label " + "group by user_id) c on c.user_id=a.id"; diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 66af5f54..9429817b 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -26,6 +26,7 @@ import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService; import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; import com.alibaba.otter.canal.client.adapter.support.*; +import org.slf4j.MDC; /** * HBase外部适配器 @@ -59,6 +60,7 @@ public class HbaseAdapter implements OuterAdapter { @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { try { + MDC.put("adapter", "hbase"); this.envProperties = envProperties; Map hbaseMappingTmp = MappingConfigLoader.load(envProperties); // 过滤不匹配的key的配置 @@ -222,6 +224,7 @@ public class HbaseAdapter implements OuterAdapter { } catch (IOException e) { throw new RuntimeException(e); } + MDC.remove("adapter"); } @Override diff --git a/client-adapter/launcher/src/main/resources/application.yml b/client-adapter/launcher/src/main/resources/application.yml index f08b1df9..3be90bb6 100644 --- a/client-adapter/launcher/src/main/resources/application.yml +++ b/client-adapter/launcher/src/main/resources/application.yml @@ -1,11 +1,5 @@ server: port: 8081 -logging: - level: - org.springframework: WARN - com.alibaba.otter.canal.client.adapter.hbase: DEBUG - com.alibaba.otter.canal.client.adapter.es: DEBUG - com.alibaba.otter.canal.client.adapter.rdb: DEBUG spring: jackson: date-format: yyyy-MM-dd HH:mm:ss diff --git a/client-adapter/launcher/src/main/resources/logback.xml b/client-adapter/launcher/src/main/resources/logback.xml new file mode 100644 index 00000000..80765f80 --- /dev/null +++ b/client-adapter/launcher/src/main/resources/logback.xml @@ -0,0 +1,54 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + adapter + adapter + + + + ../logs/${adapter}/${adapter}.log + + + ../logs/${adapter}/%d{yyyy-MM-dd}/${adapter}-%d{yyyy-MM-dd}-%i.log.gz + + + 512MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index 08fd171f..46b344ac 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -5,6 +5,7 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; @@ -27,7 +28,7 @@ public class LoggerAdapterExample implements OuterAdapter { @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { - + MDC.put("adapter", "logger"); } public void sync(List dmls) { @@ -42,6 +43,6 @@ public class LoggerAdapterExample implements OuterAdapter { @Override public void destroy() { - + MDC.remove("adapter"); } } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java index 75ddfbb0..6ec5bb25 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java @@ -23,6 +23,7 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; import com.alibaba.otter.canal.client.adapter.support.*; +import org.slf4j.MDC; /** * RDB适配器实现类 @@ -67,6 +68,7 @@ public class RdbAdapter implements OuterAdapter { */ @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { + MDC.put("adapter", "rdb"); this.envProperties = envProperties; Map rdbMappingTmp = ConfigLoader.load(envProperties); // 过滤不匹配的key的配置 @@ -294,5 +296,6 @@ public class RdbAdapter implements OuterAdapter { if (dataSource != null) { dataSource.close(); } + MDC.remove("adapter"); } }