chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue #727 , support rds binlog dump

Browse Source
This commit is contained in:
七锋
2018-08-13 10:16:28 +08:00
parent fb1126cf39
commit d3f52c6d3d
35 changed files with 553 additions and 883 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogEventParserProxyTest.java
View File

@@ -8,6 +8,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogEventParserProxy;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
@@ -28,10 +29,9 @@ public class RdsBinlogEventParserProxyTest {
private static final String MYSQL_ADDRESS = "";
private static final String USERNAME = "";
private static final String PASSWORD = "";
public static final String DBNAME = "";
public static final String TBNAME = "";
public static final String DDL = "";
public static final String DBNAME = "";
public static final String TBNAME = "";
public static final String DDL = "";
@Test
public void test_timestamp() throws InterruptedException {
@@ -41,7 +41,7 @@ public class RdsBinlogEventParserProxyTest {
final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, -1);
calendar.add(Calendar.HOUR_OF_DAY, -24 * 4);
final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
controller.setSlaveId(3344L);
controller.setDetectingEnable(false);
@@ -51,13 +51,15 @@ public class RdsBinlogEventParserProxyTest {
controller.setInstanceId("");
controller.setAccesskey("");
controller.setSecretkey("");
controller.setBatchSize(4);
// controller.setRdsOpenApiUrl("https://rds.aliyuncs.com/");
controller.setDirectory("/tmp/binlog");
controller.setEventBlackFilter(new AviaterRegexFilter("mysql\\.*"));
controller.setFilterTableError(true);
controller.setBatchFileSize(4);
controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {
@Override
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
throws CanalSinkException {
throws CanalSinkException {
for (CanalEntry.Entry entry : entrys) {
if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
entryCount.incrementAndGet();
@@ -79,8 +81,8 @@ public class RdsBinlogEventParserProxyTest {
controller.setLogPositionManager(new AbstractLogPositionManager() {
private LogPosition logPosition;
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
this.logPosition = logPosition;
}
@@ -105,7 +107,6 @@ public class RdsBinlogEventParserProxyTest {
Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}
// ======================== helper method =======================
private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {

27
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java
View File

@@ -1,11 +1,14 @@
package com.alibaba.otter.canal.parse.inbound.mysql;
import java.io.File;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang.time.DateUtils;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile;
import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy;
/**
* @author agapple 2017年10月15日 下午2:14:34
@@ -13,15 +16,19 @@ import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi;
*/
public class RdsBinlogOpenApiTest {
@Test
public void testSimple() throws Throwable {
Date startTime = DateUtils.parseDate("2017-10-13 20:56:58", new String[] { "yyyy-MM-dd HH:mm:ss" });
Date endTime = DateUtils.parseDate("2017-10-14 02:57:59", new String[] { "yyyy-MM-dd HH:mm:ss" });
RdsBinlogOpenApi.downloadBinlogFiles("https://rds.aliyuncs.com/",
"",
"",
"rm-bp180v4mfjnm157es",
startTime,
endTime,
new File("/tmp/binlog/"));
Date startTime = DateUtils.parseDate("2018-08-10 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" });
Date endTime = DateUtils.parseDate("2018-08-11 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" });
String url = "https://rds.aliyuncs.com/";
String ak = "";
String sk = "";
String dbInstanceId = "";
RdsBackupPolicy backupPolicy = RdsBinlogOpenApi.queryBinlogBackupPolicy(url, ak, sk, dbInstanceId);
System.out.println(backupPolicy);
List<BinlogFile> binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime);
System.out.println(binlogFiles);
}
}