format

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/ESAdapter.java

@@ -1,6 +1,5 @@
 package com.alibaba.otter.canal.client.adapter.es.core;
 
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -8,7 +7,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import org.apache.commons.lang.StringUtils;
 
 import com.alibaba.druid.pool.DruidDataSource;
@@ -19,6 +17,7 @@ import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.monitor.ESConfigMonitor;
 import com.alibaba.otter.canal.client.adapter.es.core.service.ESSyncService;
+import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.Dml;
 import com.alibaba.otter.canal.client.adapter.support.EtlResult;
@@ -63,8 +62,8 @@ public abstract class ESAdapter implements OuterAdapter {
             // 过滤不匹配的key的配置
             esSyncConfigTmp.forEach((key, config) -> {
                 if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (config.getOuterAdapterKey() != null
-                        && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
+                    || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey()
+                        .equalsIgnoreCase(configuration.getKey()))) {
                     esSyncConfig.put(key, config);
                 }
             });
@@ -104,12 +103,12 @@ public abstract class ESAdapter implements OuterAdapter {
         String table = dml.getTable();
         Map<String, ESSyncConfig> configMap;
         if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-            configMap = dbTableEsSyncConfig
-                .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
-                     + "_" + database + "-" + table);
+            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "-"
+                                                + StringUtils.trimToEmpty(dml.getGroupId()) + "_" + database + "-"
+                                                + table);
         } else {
-            configMap = dbTableEsSyncConfig
-                .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
+            configMap = dbTableEsSyncConfig.get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-"
+                                                + table);
         }
 
         if (configMap != null && !configMap.values().isEmpty()) {
@@ -155,23 +154,29 @@ public abstract class ESAdapter implements OuterAdapter {
         }
         String schema = matcher.group(2);
 
-        schemaItem.getAliasTableItems().values().forEach(tableItem -> {
-            Map<String, ESSyncConfig> esSyncConfigMap;
-            if (envProperties != null
-                    && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                esSyncConfigMap = dbTableEsSyncConfig
-                        .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
-                                        + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-"
-                                        + tableItem.getTableName(),
-                                k -> new ConcurrentHashMap<>());
-            } else {
-                esSyncConfigMap = dbTableEsSyncConfig
-                        .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-"
-                                        + tableItem.getTableName(),
-                                k -> new ConcurrentHashMap<>());
-            }
+        schemaItem.getAliasTableItems()
+            .values()
+            .forEach(tableItem -> {
+                Map<String, ESSyncConfig> esSyncConfigMap;
+                if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
+                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+                                                                          + "-"
+                                                                          + StringUtils.trimToEmpty(config.getGroupId())
+                                                                          + "_"
+                                                                          + schema
+                                                                          + "-"
+                                                                          + tableItem.getTableName(),
+                        k -> new ConcurrentHashMap<>());
+                } else {
+                    esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination())
+                                                                          + "_"
+                                                                          + schema
+                                                                          + "-"
+                                                                          + tableItem.getTableName(),
+                        k -> new ConcurrentHashMap<>());
+                }
 
-            esSyncConfigMap.put(configName, config);
-        });
+                esSyncConfigMap.put(configName, config);
+            });
     }
 }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/config/SqlParser.java

@@ -9,8 +9,19 @@ import java.util.stream.Collectors;
 
 import com.alibaba.fastsql.sql.SQLUtils;
 import com.alibaba.fastsql.sql.ast.SQLExpr;
-import com.alibaba.fastsql.sql.ast.expr.*;
-import com.alibaba.fastsql.sql.ast.statement.*;
+import com.alibaba.fastsql.sql.ast.expr.SQLBinaryOpExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLCaseExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLCharExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLIdentifierExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLMethodInvokeExpr;
+import com.alibaba.fastsql.sql.ast.expr.SQLPropertyExpr;
+import com.alibaba.fastsql.sql.ast.statement.SQLExprTableSource;
+import com.alibaba.fastsql.sql.ast.statement.SQLJoinTableSource;
+import com.alibaba.fastsql.sql.ast.statement.SQLSelectGroupByClause;
+import com.alibaba.fastsql.sql.ast.statement.SQLSelectItem;
+import com.alibaba.fastsql.sql.ast.statement.SQLSelectStatement;
+import com.alibaba.fastsql.sql.ast.statement.SQLSubqueryTableSource;
+import com.alibaba.fastsql.sql.ast.statement.SQLTableSource;
 import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
 import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
 import com.alibaba.fastsql.sql.parser.ParserException;

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/monitor/ESConfigMonitor.java

@@ -1,13 +1,9 @@
 package com.alibaba.otter.canal.client.adapter.es.core.monitor;
 
 import java.io.File;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
-import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import org.apache.commons.io.filefilter.FileFilterUtils;
 import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
 import org.apache.commons.io.monitor.FileAlterationMonitor;
@@ -15,12 +11,9 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
+import com.alibaba.otter.canal.client.adapter.es.core.ESAdapter;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
 import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
 import com.alibaba.otter.canal.client.adapter.support.Util;
 
@@ -70,8 +63,11 @@ public class ESConfigMonitor {
             try {
                 // 加载新增的配置文件
                 String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                ESSyncConfig config = YmlConfigBinder
-                    .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
+                ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
+                    configContent,
+                    ESSyncConfig.class,
+                    null,
+                    envProperties);
                 if (config != null) {
                     config.validate();
                     addConfigToCache(file, config);
@@ -89,14 +85,17 @@ public class ESConfigMonitor {
             try {
                 if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
                     // 加载配置文件
-                    String configContent = MappingConfigsLoader
-                        .loadConfig(adapterName + File.separator + file.getName());
+                    String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator
+                                                                           + file.getName());
                     if (configContent == null) {
                         onFileDelete(file);
                         return;
                     }
-                    ESSyncConfig config = YmlConfigBinder
-                        .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
+                    ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null,
+                        configContent,
+                        ESSyncConfig.class,
+                        null,
+                        envProperties);
                     if (config == null) {
                         return;
                     }

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/service/ESSyncService.java

@@ -1,22 +1,26 @@
 package com.alibaba.otter.canal.client.adapter.es.core.service;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import javax.sql.DataSource;
 
-import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig;
 import com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.ColumnItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem;
 import com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.TableItem;
+import com.alibaba.otter.canal.client.adapter.es.core.config.SqlParser;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESSyncUtil;
 import com.alibaba.otter.canal.client.adapter.es.core.support.ESTemplate;
 import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;

client-adapter/escore/src/main/java/com/alibaba/otter/canal/client/adapter/es/core/support/ESSyncUtil.java

@@ -5,7 +5,13 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.sql.Blob;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.codec.binary.Base64;
 import org.joda.time.DateTime;