chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed rds binlog local

Browse Source
This commit is contained in:
agapple
2017-10-15 15:55:30 +08:00
parent e01e48ff65
commit 77342663e3
30 changed files with 1174 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java Normal file
View File

@@ -0,0 +1,27 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.io.File;
import java.util.Date;
import org.apache.commons.lang.time.DateUtils;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi;
/**
* @author agapple 2017年10月15日 下午2:14:34
* @since 1.0.25
*/
public class RdsBinlogOpenApiTest {
public void testSimple() throws Throwable {
Date startTime = DateUtils.parseDate("2017-10-13 20:56:58", new String[] { "yyyy-MM-dd HH:mm:ss" });
Date endTime = DateUtils.parseDate("2017-10-14 02:57:59", new String[] { "yyyy-MM-dd HH:mm:ss" });
RdsBinlogOpenApi.downloadBinlogFiles("https://rds.aliyuncs.com/",
"",
"",
"rm-bp180v4mfjnm157es",
startTime,
endTime,
new File("/tmp/binlog/"));
}
}

117
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsLocalBinlogDumpTest.java Normal file
View File

@@ -0,0 +1,117 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.sink.exception.CanalSinkException;
/**
* @author agapple 2017年10月15日 下午2:16:58
* @since 1.0.25
*/
public class RdsLocalBinlogDumpTest {
@Test
public void testSimple() {
String directory = "/tmp/rds";
final RdsLocalBinlogEventParser controller = new RdsLocalBinlogEventParser();
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
controller.setConnectionCharset(Charset.forName("UTF-8"));
controller.setDirectory(directory);
controller.setAccesskey("");
controller.setSecretkey("");
controller.setInstanceId("");
controller.setStartTime(1507860498350L);
controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {
public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException,
InterruptedException {
for (Entry entry : entrys) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == EntryType.TRANSACTIONEND) {
continue;
}
if (entry.getEntryType() == EntryType.ROWDATA) {
RowChange rowChage = null;
try {
rowChage = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+ entry.toString(), e);
}
EventType eventType = rowChage.getEventType();
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
entry.getHeader().getLogfileName(),
entry.getHeader().getLogfileOffset(),
entry.getHeader().getSchemaName(),
entry.getHeader().getTableName(),
eventType));
for (RowData rowData : rowChage.getRowDatasList()) {
if (eventType == EventType.DELETE) {
print(rowData.getBeforeColumnsList());
} else if (eventType == EventType.INSERT) {
print(rowData.getAfterColumnsList());
} else {
System.out.println("-------> before");
print(rowData.getBeforeColumnsList());
System.out.println("-------> after");
print(rowData.getAfterColumnsList());
}
}
}
}
return true;
}
});
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
controller.start();
try {
Thread.sleep(100 * 1000 * 1000L);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
controller.stop();
}
private void print(List<Column> columns) {
for (Column column : columns) {
System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated());
}
}
}

2
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java
View File

@@ -26,7 +26,7 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
public class TableMetaManagerTest {
@Resource
TableMetaManager tableMetaManager;
DatabaseTableMeta tableMetaManager;
@Test
public void testSimple() throws FileNotFoundException, IOException {

2
parse/src/test/resources/tsdb/derby-tsdb.xml
View File

@@ -7,7 +7,7 @@
default-autowire="byName">
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>

2
parse/src/test/resources/tsdb/h2-tsdb.xml
View File

@@ -7,7 +7,7 @@
default-autowire="byName">
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>

2
parse/src/test/resources/tsdb/mysql-tsdb.xml
View File

@@ -7,7 +7,7 @@
default-autowire="byName">
<!-- 基于db的实现 -->
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager">
<bean id="tableMetaTSDB" class="com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta">
<property name="metaHistoryDAO" ref="metaHistoryDAO"/>
<property name="metaSnapshotDAO" ref="metaSnapshotDAO"/>
</bean>