From 154ba422f2ae6eb5d819f06f5469e256b30f454c Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 13 Nov 2018 14:10:48 +0800 Subject: [PATCH] fixed plugin --- client-adapter/launcher/pom.xml | 2 +- .../launcher/src/main/assembly/dev.xml | 102 +++++++++--------- .../launcher/src/main/assembly/release.xml | 8 +- .../launcher/src/main/bin/startup.bat | 2 +- .../launcher/src/main/bin/startup.sh | 4 + .../inbound/mysql/dbsync/LogEventConvert.java | 7 +- .../inbound/mysql/dbsync/TableMetaCache.java | 12 +++ 7 files changed, 77 insertions(+), 60 deletions(-) diff --git a/client-adapter/launcher/pom.xml b/client-adapter/launcher/pom.xml index d48e6e71..df31b8c8 100644 --- a/client-adapter/launcher/pom.xml +++ b/client-adapter/launcher/pom.xml @@ -137,7 +137,7 @@ jar-with-dependencies - ${project.basedir}/target/canal-adapter/lib + ${project.basedir}/target/canal-adapter/plugin diff --git a/client-adapter/launcher/src/main/assembly/dev.xml b/client-adapter/launcher/src/main/assembly/dev.xml index 84962aec..1138f3f3 100644 --- a/client-adapter/launcher/src/main/assembly/dev.xml +++ b/client-adapter/launcher/src/main/assembly/dev.xml @@ -6,36 +6,36 @@ false - - . - / - - README* - - - - ./src/main/bin - bin - - **/* - - 0755 - - - ./src/main/resources - /conf - - **/* + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/resources + /conf + + **/* - - - - ../elasticsearch/src/main/resources/es - /conf/es - - **/* - - + + + + ../elasticsearch/src/main/resources/es + /conf/es + + **/* + + ../hbase/src/main/resources/hbase /conf/hbase @@ -43,27 +43,31 @@ **/* - - ../rdb/src/main/resources/ - /conf + + ../rdb/src/main/resources/ + /conf META-INF/** - - - target - logs - - **/* - - - - - - lib - - junit:junit - - - + + + target + logs + + **/* + + + + ${project.basedir}/target/canal-adapter/plugin + /plugin/ + + + + + lib + + junit:junit + + + diff --git a/client-adapter/launcher/src/main/assembly/release.xml b/client-adapter/launcher/src/main/assembly/release.xml index bebad929..456b4a6a 100644 --- a/client-adapter/launcher/src/main/assembly/release.xml +++ b/client-adapter/launcher/src/main/assembly/release.xml @@ -58,13 +58,9 @@ **/* - - ${project.basedir}/target/canal-adapter/lib - /lib/ - - *-jar-with-dependencies.jar - + ${project.basedir}/target/canal-adapter/plugin + /plugin/ diff --git a/client-adapter/launcher/src/main/bin/startup.bat b/client-adapter/launcher/src/main/bin/startup.bat index 0c464dff..5c834a27 100755 --- a/client-adapter/launcher/src/main/bin/startup.bat +++ b/client-adapter/launcher/src/main/bin/startup.bat @@ -8,7 +8,7 @@ if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% set conf_dir=%ENV_PATH%\..\conf set CLASSPATH=%conf_dir% -set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% +set CLASSPATH=%conf_dir%\..\lib\*;%conf_dir%\..\plugin\*;%CLASSPATH% set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 diff --git a/client-adapter/launcher/src/main/bin/startup.sh b/client-adapter/launcher/src/main/bin/startup.sh index 69993483..538733e1 100644 --- a/client-adapter/launcher/src/main/bin/startup.sh +++ b/client-adapter/launcher/src/main/bin/startup.sh @@ -53,6 +53,10 @@ ADAPTER_OPTS="-DappName=canal-adapter" for i in $base/lib/*; do CLASSPATH=$i:"$CLASSPATH"; done + +for i in $base/plugin/*; + do CLASSPATH=$i:"$CLASSPATH"; +done CLASSPATH="$base/conf:$CLASSPATH"; echo "cd to $bin_abs_path for workaround relative path" diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 5c5432c6..1ba0930d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -646,15 +646,16 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar continue; } - if (fieldMeta != null && existOptionalMetaData) { + if (fieldMeta != null && existOptionalMetaData && tableMetaCache.isOnTSDB()) { // check column info boolean check = StringUtils.equalsIgnoreCase(fieldMeta.getColumnName(), info.name); check &= (fieldMeta.isUnsigned() == info.unsigned); check &= (fieldMeta.isNullable() == info.nullable); if (!check) { - throw new CanalParseException("MySQL8.0 unmatch column metadata & pls submit issue , db : " - + fieldMeta.toString() + " , binlog : " + info.toString() + throw new CanalParseException("MySQL8.0 unmatch column metadata & pls submit issue , table : " + + tableMeta.getFullName() + ", db fieldMeta : " + + fieldMeta.toString() + " , binlog fieldMeta : " + info.toString() + " , on : " + event.getHeader().getLogFileName() + ":" + event.getHeader().getLogPos()); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index 4c1cef3c..69b8a3e9 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -39,6 +39,7 @@ public class TableMetaCache { public static final String EXTRA = "EXTRA"; private MysqlConnection connection; private boolean isOnRDS = false; + private boolean isOnTSDB = false; private TableMetaTSDB tableMetaTSDB; // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表 @@ -67,6 +68,8 @@ public class TableMetaCache { } }); + } else { + isOnTSDB = true; } try { @@ -244,6 +247,15 @@ public class TableMetaCache { .toString(); } + + public boolean isOnTSDB() { + return isOnTSDB; + } + + public void setOnTSDB(boolean isOnTSDB) { + this.isOnTSDB = isOnTSDB; + } + public boolean isOnRDS() { return isOnRDS; }