chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

使用统一log

Browse Source
This commit is contained in:
mcy
2019-03-26 11:29:52 +08:00
parent 37c5168a7d
commit a137deeaff
6 changed files with 29 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
View File

@@ -30,7 +30,6 @@ import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
import com.alibaba.otter.canal.client.adapter.support.*; import com.alibaba.otter.canal.client.adapter.support.*;
import org.slf4j.MDC;
/** /**
* ES外部适配器 * ES外部适配器
@@ -71,7 +70,6 @@ public class ESAdapter implements OuterAdapter {
@Override @Override
public void init(OuterAdapterConfig configuration, Properties envProperties) { public void init(OuterAdapterConfig configuration, Properties envProperties) {
try { try {
MDC.put("adapter", "es");
this.envProperties = envProperties; this.envProperties = envProperties;
Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties);
// 过滤不匹配的key的配置 // 过滤不匹配的key的配置
@@ -141,6 +139,7 @@ public class ESAdapter implements OuterAdapter {
} }
} }
@Override
public void sync(List<Dml> dmls) { public void sync(List<Dml> dmls) {
if (dmls == null || dmls.isEmpty()) { if (dmls == null || dmls.isEmpty()) {
return; return;
@@ -151,6 +150,7 @@ public class ESAdapter implements OuterAdapter {
} }
} }
esSyncService.commit(); // 批次统一提交 esSyncService.commit(); // 批次统一提交
} }
private void sync(Dml dml) { private void sync(Dml dml) {
@@ -241,7 +241,6 @@ public class ESAdapter implements OuterAdapter {
if (transportClient != null) { if (transportClient != null) {
transportClient.close(); transportClient.close();
} }
MDC.remove("adapter");
} }
@Override @Override

6
client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java
View File

@@ -26,7 +26,6 @@ import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseEtlService;
import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService; import com.alibaba.otter.canal.client.adapter.hbase.service.HbaseSyncService;
import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate;
import com.alibaba.otter.canal.client.adapter.support.*; import com.alibaba.otter.canal.client.adapter.support.*;
import org.slf4j.MDC;
/** /**
* HBase外部适配器 * HBase外部适配器
@@ -60,7 +59,6 @@ public class HbaseAdapter implements OuterAdapter {
@Override @Override
public void init(OuterAdapterConfig configuration, Properties envProperties) { public void init(OuterAdapterConfig configuration, Properties envProperties) {
try { try {
MDC.put("adapter", "hbase");
this.envProperties = envProperties; this.envProperties = envProperties;
Map<String, MappingConfig> hbaseMappingTmp = MappingConfigLoader.load(envProperties); Map<String, MappingConfig> hbaseMappingTmp = MappingConfigLoader.load(envProperties);
// 过滤不匹配的key的配置 // 过滤不匹配的key的配置
@@ -104,6 +102,7 @@ public class HbaseAdapter implements OuterAdapter {
} }
} }
@Override
public void sync(List<Dml> dmls) { public void sync(List<Dml> dmls) {
if (dmls == null || dmls.isEmpty()) { if (dmls == null || dmls.isEmpty()) {
return; return;
@@ -113,7 +112,7 @@ public class HbaseAdapter implements OuterAdapter {
} }
} }
public void sync(Dml dml) { private void sync(Dml dml) {
if (dml == null) { if (dml == null) {
return; return;
} }
@@ -224,7 +223,6 @@ public class HbaseAdapter implements OuterAdapter {
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
MDC.remove("adapter");
} }
@Override @Override

10
client-adapter/launcher/src/main/bin/startup.sh
View File

@@ -40,11 +40,11 @@ if [ -z "$JAVA" ]; then
fi fi
fi fi
case "$#" case "$#"
in in
0 ) 0 )
;; ;;
2 ) 2 )
if [ "$1" = "debug" ]; then if [ "$1" = "debug" ]; then
DEBUG_PORT=$2 DEBUG_PORT=$2
DEBUG_SUSPEND="n" DEBUG_SUSPEND="n"
@@ -76,8 +76,8 @@ echo "cd to $bin_abs_path for workaround relative path"
cd $bin_abs_path cd $bin_abs_path
echo CLASSPATH :$CLASSPATH echo CLASSPATH :$CLASSPATH
$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $ADAPTER_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication 1>>$base/logs/adapter.log 2>&1 & $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $ADAPTER_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication 1>>/dev/null 2>&1 &
echo $! > $base/bin/adapter.pid echo $! > $base/bin/adapter.pid
echo "cd to $current_path for continue" echo "cd to $current_path for continue"
cd $current_path cd $current_path

22
client-adapter/launcher/src/main/resources/logback.xml
View File

@@ -8,17 +8,17 @@
</appender> </appender>
<appender name="CANAL-ROOT" class="ch.qos.logback.classic.sift.SiftingAppender"> <appender name="CANAL-ROOT" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator> <discriminator>
<Key>adapter</Key> <Key>adapter</Key>
<DefaultValue>adapter</DefaultValue> <DefaultValue>adapter</DefaultValue>
</discriminator> </discriminator>
<sift> <sift>
<appender name="FILE-${adapter}" class="ch.qos.logback.core.rolling.RollingFileAppender"> <appender name="FILE-adapter" class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>../logs/${adapter}/${adapter}.log</File> <File>../logs/adapter/adapter.log</File>
<rollingPolicy <rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- rollover daily --> <!-- rollover daily -->
<fileNamePattern>../logs/${adapter}/%d{yyyy-MM-dd}/${adapter}-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern> <fileNamePattern>../logs/adapter/%d{yyyy-MM-dd}/adapter-%d{yyyy-MM-dd}-%i.log.gz</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<!-- or whenever the file size reaches 100MB --> <!-- or whenever the file size reaches 100MB -->
<maxFileSize>512MB</maxFileSize> <maxFileSize>512MB</maxFileSize>
@@ -36,14 +36,22 @@
<logger name="com.alibaba.otter.canal.client.adapter.hbase" additivity="false"> <logger name="com.alibaba.otter.canal.client.adapter.hbase" additivity="false">
<level value="DEBUG" /> <level value="DEBUG" />
<appender-ref ref="STDOUT"/>
<appender-ref ref="CANAL-ROOT" /> <appender-ref ref="CANAL-ROOT" />
</logger> </logger>
<logger name="com.alibaba.otter.canal.client.adapter.es" additivity="false"> <logger name="com.alibaba.otter.canal.client.adapter.es" additivity="false">
<level value="DEBUG" /> <level value="DEBUG" />
<appender-ref ref="STDOUT"/>
<appender-ref ref="CANAL-ROOT" /> <appender-ref ref="CANAL-ROOT" />
</logger> </logger>
<logger name="com.alibaba.otter.canal.client.adapter.rdb" additivity="false"> <logger name="com.alibaba.otter.canal.client.adapter.rdb" additivity="false">
<level value="DEBUG" /> <level value="DEBUG" />
<appender-ref ref="STDOUT"/>
<appender-ref ref="CANAL-ROOT" />
</logger>
<logger name="com.alibaba.otter.canal.client.adapter.logger" additivity="false">
<level value="DEBUG" />
<appender-ref ref="STDOUT"/>
<appender-ref ref="CANAL-ROOT" /> <appender-ref ref="CANAL-ROOT" />
</logger> </logger>

3
client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java
View File

@@ -5,7 +5,6 @@ import java.util.Properties;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.serializer.SerializerFeature;
@@ -28,7 +27,6 @@ public class LoggerAdapterExample implements OuterAdapter {
@Override @Override
public void init(OuterAdapterConfig configuration, Properties envProperties) { public void init(OuterAdapterConfig configuration, Properties envProperties) {
MDC.put("adapter", "logger");
} }
public void sync(List<Dml> dmls) { public void sync(List<Dml> dmls) {
@@ -43,6 +41,5 @@ public class LoggerAdapterExample implements OuterAdapter {
@Override @Override
public void destroy() { public void destroy() {
MDC.remove("adapter");
} }
} }

18
client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java
View File

@@ -14,7 +14,6 @@ import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.OuterAdapter;
@@ -26,12 +25,7 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService;
import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService;
import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; import com.alibaba.otter.canal.client.adapter.support.*;
import com.alibaba.otter.canal.client.adapter.support.Dml;
import com.alibaba.otter.canal.client.adapter.support.EtlResult;
import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
import com.alibaba.otter.canal.client.adapter.support.SPI;
import com.alibaba.otter.canal.client.adapter.support.Util;
/** /**
* RDB适配器实现类 * RDB适配器实现类
@@ -76,14 +70,13 @@ public class RdbAdapter implements OuterAdapter {
*/ */
@Override @Override
public void init(OuterAdapterConfig configuration, Properties envProperties) { public void init(OuterAdapterConfig configuration, Properties envProperties) {
MDC.put("adapter", "rdb");
this.envProperties = envProperties; this.envProperties = envProperties;
Map<String, MappingConfig> rdbMappingTmp = ConfigLoader.load(envProperties); Map<String, MappingConfig> rdbMappingTmp = ConfigLoader.load(envProperties);
// 过滤不匹配的key的配置 // 过滤不匹配的key的配置
rdbMappingTmp.forEach((key, mappingConfig) -> { rdbMappingTmp.forEach((key, mappingConfig) -> {
if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null)
|| (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey() || (mappingConfig.getOuterAdapterKey() != null
.equalsIgnoreCase(configuration.getKey()))) { && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
rdbMapping.put(key, mappingConfig); rdbMapping.put(key, mappingConfig);
} }
}); });
@@ -143,8 +136,8 @@ public class RdbAdapter implements OuterAdapter {
String threads = properties.get("threads"); String threads = properties.get("threads");
// String commitSize = properties.get("commitSize"); // String commitSize = properties.get("commitSize");
boolean skipDupException = BooleanUtils.toBoolean(configuration.getProperties() boolean skipDupException = BooleanUtils
.getOrDefault("skipDupException", "true")); .toBoolean(configuration.getProperties().getOrDefault("skipDupException", "true"));
rdbSyncService = new RdbSyncService(dataSource, rdbSyncService = new RdbSyncService(dataSource,
threads != null ? Integer.valueOf(threads) : null, threads != null ? Integer.valueOf(threads) : null,
skipDupException); skipDupException);
@@ -304,6 +297,5 @@ public class RdbAdapter implements OuterAdapter {
if (dataSource != null) { if (dataSource != null) {
dataSource.close(); dataSource.close();
} }
MDC.remove("adapter");
} }
} }