Compare commits

..

10 Commits

Author SHA1 Message Date
agapple
20942533e6 [maven-release-plugin] prepare for next development iteration 2017-12-04 14:43:29 +08:00
agapple
6b2d80a145 [maven-release-plugin] prepare release canal-1.0.25 2017-12-04 14:43:16 +08:00
agapple
01ef15223c fixed issue #415, ignore flush privileges 2017-12-04 13:19:58 +08:00
agapple
da3290c381 Merge pull request #424 from lcybo/master
1.0.23里面修复的KILL DUMP exception问题在(#334)里面被覆盖回去了
2017-12-03 21:36:33 -06:00
agapple
a9284b1b39 fixed compiler error 2017-12-04 11:35:02 +08:00
Chuanyi Li
3409472eb9 kill CONNECTION exception fix revert. 2017-11-23 22:25:47 +08:00
agapple
5348994b4a fixed alisql heartbeat 2017-11-15 10:42:58 +08:00
agapple
a3b9f6f1eb fixed NPE 2017-11-14 23:44:15 +08:00
agapple
2dc019709b fixed quota ddl 2017-11-14 23:31:46 +08:00
agapple
c56f58deae upgrade druid to 1.1.5 2017-11-01 20:29:47 +08:00
23 changed files with 57 additions and 28 deletions

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -1,10 +1,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<artifactId>canal.common</artifactId> <artifactId>canal.common</artifactId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -101,7 +101,8 @@ public class MysqlConnector {
MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector); MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector);
executor.update("KILL CONNECTION " + connectionId); executor.update("KILL CONNECTION " + connectionId);
} catch (Exception e) { } catch (Exception e) {
throw new IOException("KILL DUMP " + connectionId + " failure", e); // 忽略具体异常
logger.info("KILL DUMP " + connectionId + " failure", e);
} finally { } finally {
if (connector != null) { if (connector != null) {
connector.disconnect(); connector.disconnect();

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath> <relativePath>../../pom.xml</relativePath>
</parent> </parent>
<artifactId>canal.instance.core</artifactId> <artifactId>canal.instance.core</artifactId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath> <relativePath>../../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath> <relativePath>../../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -1,10 +1,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<artifactId>canal.parse</artifactId> <artifactId>canal.parse</artifactId>

View File

@@ -364,6 +364,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
throw new TableIdNotFoundException("not found tableId:" + event.getTableId()); throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
} }
boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName());
boolean isRDSHeartBeat = tableMetaCache.isOnRDS() boolean isRDSHeartBeat = tableMetaCache.isOnRDS()
&& isRDSHeartBeat(table.getDbName(), table.getTableName()); && isRDSHeartBeat(table.getDbName(), table.getTableName());
@@ -387,6 +388,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0"); FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0");
FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0"); FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0");
tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta)); tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
} else if (isHeartBeat) {
// 处理alisql模式的test.heartbeat心跳数据
// 心跳表基本无权限,需要mock一个tableMeta
FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null);
FieldMeta typeMeta = new FieldMeta("type", "int(11)", true, false, null);
tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
} }
EventType eventType = null; EventType eventType = null;
@@ -767,6 +774,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|| "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType); || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
} }
private boolean isAliSQLHeartBeat(String schema, String table) {
return "test".equalsIgnoreCase(schema) && "heartbeat".equalsIgnoreCase(table);
}
private boolean isRDSHeartBeat(String schema, String table) { private boolean isRDSHeartBeat(String schema, String table) {
return "mysql".equalsIgnoreCase(schema) && "ha_health_check".equalsIgnoreCase(table); return "mysql".equalsIgnoreCase(schema) && "ha_health_check".equalsIgnoreCase(table);
} }

View File

@@ -14,6 +14,7 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser;
import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB;
import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheBuilder;
@@ -103,7 +104,9 @@ public class TableMetaCache {
* size), * size),
"YES")); "YES"));
meta.setKey("PRI".equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size))); meta.setKey("PRI".equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size)));
meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size)); // 特殊处理引号
meta.setDefaultValue(DruidDdlParser.unescapeQuotaName(packet.getFieldValues()
.get(nameMaps.get(COLUMN_DEFAULT) + i * size)));
meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size)); meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size));
result.add(meta); result.add(meta);

View File

@@ -187,7 +187,7 @@ public class DruidDdlParser {
} }
public static String unescapeName(String name) { public static String unescapeName(String name) {
if (name.length() > 2) { if (name != null && name.length() > 2) {
char c0 = name.charAt(0); char c0 = name.charAt(0);
char x0 = name.charAt(name.length() - 1); char x0 = name.charAt(name.length() - 1);
if ((c0 == '"' && x0 == '"') || (c0 == '`' && x0 == '`')) { if ((c0 == '"' && x0 == '"') || (c0 == '`' && x0 == '`')) {
@@ -198,4 +198,16 @@ public class DruidDdlParser {
return name; return name;
} }
public static String unescapeQuotaName(String name) {
if (name != null && name.length() > 2) {
char c0 = name.charAt(0);
char x0 = name.charAt(name.length() - 1);
if (c0 == '\'' && x0 == '\'') {
return name.substring(1, name.length() - 1);
}
}
return name;
}
} }

View File

@@ -64,7 +64,10 @@ public class MemoryTableMeta implements TableMetaTSDB {
} }
try { try {
repository.console(ddl); // druid暂时flush privileges语法解析有问题
if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")) {
repository.console(ddl);
}
} catch (Throwable e) { } catch (Throwable e) {
logger.warn("parse faield : " + ddl, e); logger.warn("parse faield : " + ddl, e);
} }
@@ -187,7 +190,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
if (column.getDefaultExpr() == null || column.getDefaultExpr() instanceof SQLNullExpr) { if (column.getDefaultExpr() == null || column.getDefaultExpr() instanceof SQLNullExpr) {
fieldMeta.setDefaultValue(null); fieldMeta.setDefaultValue(null);
} else { } else {
fieldMeta.setDefaultValue(getSqlName(column.getDefaultExpr())); fieldMeta.setDefaultValue(DruidDdlParser.unescapeQuotaName(getSqlName(column.getDefaultExpr())));
} }
fieldMeta.setColumnName(name); fieldMeta.setColumnName(name);
@@ -233,6 +236,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
} }
} }
public SchemaRepository getRepository() { public SchemaRepository getRepository() {
return repository; return repository;
} }

View File

@@ -4,7 +4,7 @@
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>canal module for otter ${project.version}</name> <name>canal module for otter ${project.version}</name>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<url>https://github.com/alibaba/canal</url> <url>https://github.com/alibaba/canal</url>
<parent> <parent>
<groupId>org.sonatype.oss</groupId> <groupId>org.sonatype.oss</groupId>
@@ -254,7 +254,7 @@
<dependency> <dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>druid</artifactId> <artifactId>druid</artifactId>
<version>1.1.5-preview_14</version> <version>1.1.6</version>
</dependency> </dependency>
<!-- log --> <!-- log -->
<dependency> <dependency>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<artifactId>canal.server</artifactId> <artifactId>canal.server</artifactId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>

View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId> <artifactId>canal</artifactId>
<version>1.0.25-SNAPSHOT</version> <version>1.0.26-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
<groupId>com.alibaba.otter</groupId> <groupId>com.alibaba.otter</groupId>