chenjl/canal
1
0
Fork 0
mirror of https://github.com/alibaba/canal.git synced 2026-11-04 00:30:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed code template and format

Browse Source
This commit is contained in:
agapple
2015-04-28 21:11:41 +08:00
parent ea889cb004
commit 7ebb3fd1ab
86 changed files with 2763 additions and 3554 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java
View File

@@ -44,7 +44,9 @@ public class CanalConnectors {
*/
public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
String username, String password) {
ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
password,
destination,
new SimpleNodeAccessStrategy(addresses));
canalConnector.setSoTimeout(30 * 1000);
return canalConnector;
@@ -61,13 +63,10 @@ public class CanalConnectors {
*/
public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
String password) {
ClusterCanalConnector canalConnector = new ClusterCanalConnector(
username,
ClusterCanalConnector canalConnector = new ClusterCanalConnector(username,
password,
destination,
new ClusterNodeAccessStrategy(
destination,
ZkClientx.getZkClient(zkServers)));
new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkServers)));
canalConnector.setSoTimeout(30 * 1000);
return canalConnector;
}

1
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java
View File

@@ -411,7 +411,6 @@ public class SimpleCanalConnector implements CanalConnector {
}
}
private void waitClientRunning() {
try {
if (zkClientx != null) {

291
codeformat.xml Normal file
View File

@@ -0,0 +1,291 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<profiles version="12">
<profile kind="CodeFormatterProfile" name="canal-format" version="12">
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="84"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
<setting id="org.eclipse.jdt.core.compiler.source" value="1.7"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="84"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="84"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="120"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="84"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.7"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="84"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.7"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="18"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="true"/>
</profile>
</profiles>

27
codetemplates.xml Normal file
View File

@@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?><templates><template autoinsert="true" context="gettercomment_context" deleted="false" description="Comment for getter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name="gettercomment">/**
* @return the ${bare_field_name}
*/</template><template autoinsert="true" context="settercomment_context" deleted="false" description="Comment for setter method" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.settercomment" name="settercomment">/**
* @param ${param} the ${bare_field_name} to set
*/</template><template autoinsert="true" context="constructorcomment_context" deleted="false" description="Comment for created constructors" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name="constructorcomment">/**
* ${tags}
*/</template><template autoinsert="false" context="filecomment_context" deleted="false" description="Comment for created Java files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.filecomment" name="filecomment"/><template autoinsert="false" context="typecomment_context" deleted="false" description="Comment for created types" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.typecomment" name="typecomment">/**
* @author ${user} ${date} ${time}
* @since 1.0.0
*/</template><template autoinsert="true" context="fieldcomment_context" deleted="false" description="Comment for fields" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name="fieldcomment">/**
*
*/</template><template autoinsert="true" context="methodcomment_context" deleted="false" description="Comment for non-overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name="methodcomment">/**
* ${tags}
*/</template><template autoinsert="false" context="overridecomment_context" deleted="false" description="Comment for overriding methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name="overridecomment"/><template autoinsert="true" context="delegatecomment_context" deleted="false" description="Comment for delegate methods" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name="delegatecomment">/**
* ${tags}
* ${see_to_target}
*/</template><template autoinsert="false" context="newtype_context" deleted="false" description="Newly created files" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.newtype" name="newtype">${filecomment}
${package_declaration}
${typecomment}
${type_declaration}</template><template autoinsert="true" context="classbody_context" deleted="false" description="Code in new class type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.classbody" name="classbody">
</template><template autoinsert="true" context="interfacebody_context" deleted="false" description="Code in new interface type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name="interfacebody">
</template><template autoinsert="true" context="enumbody_context" deleted="false" description="Code in new enum type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.enumbody" name="enumbody">
</template><template autoinsert="true" context="annotationbody_context" deleted="false" description="Code in new annotation type bodies" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name="annotationbody">
</template><template autoinsert="true" context="catchblock_context" deleted="false" description="Code in new catch blocks" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.catchblock" name="catchblock">// ${todo} Auto-generated catch block
${exception_var}.printStackTrace();</template><template autoinsert="true" context="methodbody_context" deleted="false" description="Code in created method stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.methodbody" name="methodbody">// ${todo} Auto-generated method stub
${body_statement}</template><template autoinsert="true" context="constructorbody_context" deleted="false" description="Code in created constructor stubs" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name="constructorbody">${body_statement}
// ${todo} Auto-generated constructor stub</template><template autoinsert="true" context="getterbody_context" deleted="false" description="Code in created getters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.getterbody" name="getterbody">return ${field};</template><template autoinsert="true" context="setterbody_context" deleted="false" description="Code in created setters" enabled="true" id="org.eclipse.jdt.ui.text.codetemplates.setterbody" name="setterbody">${field} = ${param};</template></templates>

3
common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java
View File

@@ -38,7 +38,8 @@ public class AddressUtils {
private static boolean isValidHostAddress(InetAddress address) {
if (address == null || address.isLoopbackAddress()) return false;
String name = address.getHostAddress();
return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches());
return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name)
.matches());
}
public static String getHostIp() {

39
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.common.zookeeper;
import com.google.common.collect.MigrateMap;
import java.util.Map;
import org.I0Itec.zkclient.IZkConnection;
@@ -13,7 +12,7 @@ import org.I0Itec.zkclient.serialize.ZkSerializer;
import org.apache.zookeeper.CreateMode;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 使用自定义的ZooKeeperx for zk connection
@@ -24,11 +23,9 @@ import com.google.common.collect.MapMaker;
public class ZkClientx extends ZkClient {
// 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>()
{
private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>() {
public ZkClientx apply(String servers)
{
public ZkClientx apply(String servers) {
return new ZkClientx(servers);
}
});
@@ -65,10 +62,12 @@ public class ZkClientx extends ZkClient {
* Create a persistent Sequential node.
*
* @param path
* @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
* in case the path already exists
* @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
* @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
* @param createParents if true all parent dirs are created as well and no
* {@link ZkNodeExistsException} is thrown in case the path already exists
* @throws ZkInterruptedException if operation was interrupted, or a
* required reconnection got interrupted
* @throws IllegalArgumentException if called from anything except the
* ZooKeeper event thread
* @throws ZkException if any ZooKeeper exception occurred
* @throws RuntimeException if any other exception occurs
*/
@@ -92,10 +91,12 @@ public class ZkClientx extends ZkClient {
*
* @param path
* @param data
* @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
* in case the path already exists
* @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
* @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
* @param createParents if true all parent dirs are created as well and no
* {@link ZkNodeExistsException} is thrown in case the path already exists
* @throws ZkInterruptedException if operation was interrupted, or a
* required reconnection got interrupted
* @throws IllegalArgumentException if called from anything except the
* ZooKeeper event thread
* @throws ZkException if any ZooKeeper exception occurred
* @throws RuntimeException if any other exception occurs
*/
@@ -121,10 +122,12 @@ public class ZkClientx extends ZkClient {
*
* @param path
* @param data
* @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
* in case the path already exists
* @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
* @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
* @param createParents if true all parent dirs are created as well and no
* {@link ZkNodeExistsException} is thrown in case the path already exists
* @throws ZkInterruptedException if operation was interrupted, or a
* required reconnection got interrupted
* @throws IllegalArgumentException if called from anything except the
* ZooKeeper event thread
* @throws ZkException if any ZooKeeper exception occurred
* @throws RuntimeException if any other exception occurs
*/

5
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java
View File

@@ -15,8 +15,8 @@ import org.apache.zookeeper.ClientCnxn;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.ZooKeeper.States;
import org.apache.zookeeper.client.ConnectStringParser;
import org.apache.zookeeper.client.HostProvider;
@@ -159,8 +159,7 @@ public class ZooKeeperx implements IZkConnection {
// 强制获取zk中的地址信息
ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(
serverAddressesField,
List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(serverAddressesField,
hostProvider);
// 添加第二组集群列表
serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());

6
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java
View File

@@ -120,8 +120,10 @@ public class ZookeeperPathUtils {
}
public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) {
return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName,
String.valueOf(clientId), getBatchMarkNode(batchId));
return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH,
destinationName,
String.valueOf(clientId),
getBatchMarkNode(batchId));
}
public static String getCursorPath(String destination, short clientId) {

3
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java
View File

@@ -11,7 +11,8 @@ import java.util.Map;
public class ServerRunningMonitors {
private static ServerRunningData serverData;
private static Map runningMonitors; // <String, ServerRunningMonitor>
private static Map runningMonitors; // <String,
// ServerRunningMonitor>
public static ServerRunningData getServerData() {
return serverData;

14
common/src/main/java/com/google/common/collect/MigrateMap.java
View File

@@ -1,17 +1,17 @@
package com.google.common.collect;
import com.google.common.base.Function;
import java.util.concurrent.ConcurrentMap;
public class MigrateMap
{
public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker, Function<? super K, ? extends V> computingFunction)
{
import com.google.common.base.Function;
public class MigrateMap {
public static <K, V> ConcurrentMap<K, V> makeComputingMap(MapMaker maker,
Function<? super K, ? extends V> computingFunction) {
return maker.makeComputingMap(computingFunction);
}
public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction)
{
public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
return new MapMaker().makeComputingMap(computingFunction);
}
}

97
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java
View File

@@ -6,24 +6,24 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* An utility class implements MySQL/Java charsets conversion. you can see com.mysql.jdbc.CharsetMapping.
* An utility class implements MySQL/Java charsets conversion. you can see
* com.mysql.jdbc.CharsetMapping.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
*/
public final class CharsetConversion
{
public final class CharsetConversion {
static final Log logger = LogFactory.getLog(CharsetConversion.class);
static final class Entry
{
static final class Entry {
protected final int charsetId;
protected final String mysqlCharset;
protected final String mysqlCollation;
protected final String javaCharset;
Entry(final int id, String mysqlCharset, // NL
String mysqlCollation, String javaCharset)
{
String mysqlCollation, String javaCharset){
this.charsetId = id;
this.mysqlCharset = mysqlCharset;
this.mysqlCollation = mysqlCollation;
@@ -34,38 +34,31 @@ public final class CharsetConversion
// Character set data used in lookups. The array will be sparse.
static final Entry[] entries = new Entry[0xff];
static Entry getEntry(final int id)
{
if (id >= 0 && id < entries.length)
{
static Entry getEntry(final int id) {
if (id >= 0 && id < entries.length) {
return entries[id];
}
else
{
} else {
throw new IllegalArgumentException("Invalid charset id: " + id);
}
}
// Loads character set information.
static void putEntry(final int charsetId, String mysqlCharset,
String mysqlCollation, String javaCharset)
{
static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation, String javaCharset) {
entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
mysqlCollation, javaCharset);
mysqlCollation,
javaCharset);
}
// Loads character set information.
@Deprecated
static void putEntry(final int charsetId, String mysqlCharset,
String mysqlCollation)
{
static void putEntry(final int charsetId, String mysqlCharset, String mysqlCollation) {
entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
mysqlCollation, /* Unknown java charset */null);
mysqlCollation, /* Unknown java charset */
null);
}
// Load character set data statically.
static
{
static {
putEntry(1, "big5", "big5_chinese_ci", "Big5");
putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
@@ -274,16 +267,12 @@ public final class CharsetConversion
/**
* Return defined charset name for mysql.
*/
public static String getCharset(final int id)
{
public static String getCharset(final int id) {
Entry entry = getEntry(id);
if (entry != null)
{
if (entry != null) {
return entry.mysqlCharset;
}
else
{
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
@@ -292,16 +281,12 @@ public final class CharsetConversion
/**
* Return defined collaction name for mysql.
*/
public static String getCollation(final int id)
{
public static String getCollation(final int id) {
Entry entry = getEntry(id);
if (entry != null)
{
if (entry != null) {
return entry.mysqlCollation;
}
else
{
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
@@ -310,55 +295,41 @@ public final class CharsetConversion
/**
* Return converted charset name for java.
*/
public static String getJavaCharset(final int id)
{
public static String getJavaCharset(final int id) {
Entry entry = getEntry(id);
if (entry != null)
{
if (entry.javaCharset != null)
{
if (entry != null) {
if (entry.javaCharset != null) {
return entry.javaCharset;
}
else
{
logger.warn("Unknown java charset for: id = " + id
+ ", name = " + entry.mysqlCharset + ", coll = "
} else {
logger.warn("Unknown java charset for: id = " + id + ", name = " + entry.mysqlCharset + ", coll = "
+ entry.mysqlCollation);
return null;
}
}
else
{
} else {
logger.warn("Unexpect mysql charset: " + id);
return null;
}
}
public static void main(String[] args)
{
for (int i = 0; i < entries.length; i++)
{
public static void main(String[] args) {
for (int i = 0; i < entries.length; i++) {
Entry entry = entries[i];
System.out.print(i);
System.out.print(',');
System.out.print(' ');
if (entry != null)
{
if (entry != null) {
System.out.print(entry.mysqlCharset);
System.out.print(',');
System.out.print(' ');
System.out.print(entry.javaCharset);
if (entry.javaCharset != null)
{
if (entry.javaCharset != null) {
System.out.print(',');
System.out.print(' ');
System.out.print(Charset.forName(entry.javaCharset).name());
}
}
else
{
} else {
System.out.print("null");
}
System.out.println();

281
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java
View File

@@ -21,16 +21,13 @@ import org.apache.commons.logging.LogFactory;
* DirectLogFetcher fetcher = new DirectLogFetcher();
* fetcher.open(conn, file, 0, 13);
*
* while (fetcher.fetch())
* {
* while (fetcher.fetch()) {
* LogEvent event;
* do
* {
* do {
* event = decoder.decode(fetcher, context);
*
* // process log event.
* }
* while (event != null);
* } while (event != null);
* }
* // connection closed.
* </pre>
@@ -38,8 +35,8 @@ import org.apache.commons.logging.LogFactory;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class DirectLogFetcher extends LogFetcher
{
public final class DirectLogFetcher extends LogFetcher {
protected static final Log logger = LogFactory.getLog(DirectLogFetcher.class);
/** Command to dump binlog */
@@ -64,71 +61,53 @@ public final class DirectLogFetcher extends LogFetcher
private OutputStream mysqlOutput;
private InputStream mysqlInput;
public DirectLogFetcher()
{
public DirectLogFetcher(){
super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
}
public DirectLogFetcher(final int initialCapacity)
{
public DirectLogFetcher(final int initialCapacity){
super(initialCapacity, DEFAULT_GROWTH_FACTOR);
}
public DirectLogFetcher(final int initialCapacity, final float growthFactor)
{
public DirectLogFetcher(final int initialCapacity, final float growthFactor){
super(initialCapacity, growthFactor);
}
private static final Object unwrapConnection(Object conn, Class<?> connClazz)
throws IOException
{
while (!connClazz.isInstance(conn))
{
try
{
private static final Object unwrapConnection(Object conn, Class<?> connClazz) throws IOException {
while (!connClazz.isInstance(conn)) {
try {
Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
if (connProxy.isInstance(conn))
{
if (connProxy.isInstance(conn)) {
conn = invokeMethod(conn, connProxy, "getTargetConnection");
continue;
}
}
catch (ClassNotFoundException e)
{
// org.springframework.jdbc.datasource.ConnectionProxy not found.
} catch (ClassNotFoundException e) {
// org.springframework.jdbc.datasource.ConnectionProxy not
// found.
}
try
{
try {
Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
if (connProxy.isInstance(conn))
{
if (connProxy.isInstance(conn)) {
conn = getDeclaredField(conn, connProxy, "_conn");
continue;
}
}
catch (ClassNotFoundException e)
{
} catch (ClassNotFoundException e) {
// org.apache.commons.dbcp.DelegatingConnection not found.
}
try
{
if (conn instanceof java.sql.Wrapper)
{
try {
if (conn instanceof java.sql.Wrapper) {
Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
conn = ((java.sql.Wrapper) conn).unwrap(connIface);
continue;
}
}
catch (ClassNotFoundException e)
{
} catch (ClassNotFoundException e) {
// com.mysql.jdbc.Connection not found.
}
catch (SQLException e)
{
logger.warn("Unwrap " + conn.getClass().getName() + " to "
+ connClazz.getName() + " failed: " + e.getMessage(), e);
} catch (SQLException e) {
logger.warn("Unwrap " + conn.getClass().getName() + " to " + connClazz.getName() + " failed: "
+ e.getMessage(),
e);
}
return null;
@@ -136,127 +115,85 @@ public final class DirectLogFetcher extends LogFetcher
return conn;
}
private static final Object invokeMethod(Object obj, Class<?> objClazz,
String name)
{
try
{
private static final Object invokeMethod(Object obj, Class<?> objClazz, String name) {
try {
Method method = objClazz.getMethod(name, (Class<?>[]) null);
return method.invoke(obj, (Object[]) null);
}
catch (NoSuchMethodException e)
{
throw new IllegalArgumentException("No such method: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
catch (IllegalAccessException e)
{
throw new IllegalArgumentException("Cannot invoke method: \'"
+ name + "\' @ " + objClazz.getName(), e);
}
catch (InvocationTargetException e)
{
throw new IllegalArgumentException("Invoke method failed: \'"
+ name + "\' @ " + objClazz.getName(),
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException("No such method: \'" + name + "\' @ " + objClazz.getName(), e);
} catch (IllegalAccessException e) {
throw new IllegalArgumentException("Cannot invoke method: \'" + name + "\' @ " + objClazz.getName(), e);
} catch (InvocationTargetException e) {
throw new IllegalArgumentException("Invoke method failed: \'" + name + "\' @ " + objClazz.getName(),
e.getTargetException());
}
}
private static final Object getDeclaredField(Object obj, Class<?> objClazz,
String name)
{
try
{
private static final Object getDeclaredField(Object obj, Class<?> objClazz, String name) {
try {
Field field = objClazz.getDeclaredField(name);
field.setAccessible(true);
return field.get(obj);
}
catch (NoSuchFieldException e)
{
throw new IllegalArgumentException("No such field: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
catch (IllegalAccessException e)
{
throw new IllegalArgumentException("Cannot get field: \'" + name
+ "\' @ " + objClazz.getName(), e);
} catch (NoSuchFieldException e) {
throw new IllegalArgumentException("No such field: \'" + name + "\' @ " + objClazz.getName(), e);
} catch (IllegalAccessException e) {
throw new IllegalArgumentException("Cannot get field: \'" + name + "\' @ " + objClazz.getName(), e);
}
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final int serverId)
throws IOException
{
public void open(Connection conn, String fileName, final int serverId) throws IOException {
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final int serverId,
boolean nonBlocking) throws IOException
{
public void open(Connection conn, String fileName, final int serverId, boolean nonBlocking) throws IOException {
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final long filePosition,
final int serverId) throws IOException
{
public void open(Connection conn, String fileName, final long filePosition, final int serverId) throws IOException {
open(conn, fileName, filePosition, serverId, false);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, long filePosition,
final int serverId, boolean nonBlocking) throws IOException
{
try
{
public void open(Connection conn, String fileName, long filePosition, final int serverId, boolean nonBlocking)
throws IOException {
try {
this.conn = conn;
Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
Object unwrapConn = unwrapConnection(conn, connClazz);
if (unwrapConn == null)
{
throw new IOException("Unable to unwrap "
+ conn.getClass().getName()
if (unwrapConn == null) {
throw new IOException("Unable to unwrap " + conn.getClass().getName()
+ " to com.mysql.jdbc.ConnectionImpl");
}
// Get underlying IO streams for network communications.
Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
if (connIo == null)
{
throw new IOException("Get null field:"
+ conn.getClass().getName() + "#io");
if (connIo == null) {
throw new IOException("Get null field:" + conn.getClass().getName() + "#io");
}
mysqlOutput = (OutputStream) getDeclaredField(connIo,
connIo.getClass(), "mysqlOutput");
mysqlInput = (InputStream) getDeclaredField(connIo,
connIo.getClass(), "mysqlInput");
mysqlOutput = (OutputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlOutput");
mysqlInput = (InputStream) getDeclaredField(connIo, connIo.getClass(), "mysqlInput");
if (filePosition == 0)
filePosition = BIN_LOG_HEADER_SIZE;
if (filePosition == 0) filePosition = BIN_LOG_HEADER_SIZE;
sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
position = 0;
}
catch (IOException e)
{
} catch (IOException e) {
close(); /* Do cleanup */
logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
+ ", position = " + filePosition);
logger.error("Error on COM_BINLOG_DUMP: file = " + fileName + ", position = " + filePosition);
throw e;
}
catch (ClassNotFoundException e)
{
} catch (ClassNotFoundException e) {
close(); /* Do cleanup */
throw new IOException(
"Unable to load com.mysql.jdbc.ConnectionImpl", e);
throw new IOException("Unable to load com.mysql.jdbc.ConnectionImpl", e);
}
}
@@ -265,8 +202,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param b the byte to put in the buffer
*/
protected final void putByte(byte b)
{
protected final void putByte(byte b) {
ensureCapacity(position + 1);
buffer[position++] = b;
@@ -277,8 +213,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param i16 the integer to put in the buffer
*/
protected final void putInt16(int i16)
{
protected final void putInt16(int i16) {
ensureCapacity(position + 2);
byte[] buf = buffer;
@@ -291,8 +226,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param i32 the integer to put in the buffer
*/
protected final void putInt32(long i32)
{
protected final void putInt32(long i32) {
ensureCapacity(position + 4);
byte[] buf = buffer;
@@ -307,8 +241,7 @@ public final class DirectLogFetcher extends LogFetcher
*
* @param s the value to put in the buffer
*/
protected final void putString(String s)
{
protected final void putString(String s) {
ensureCapacity(position + (s.length() * 2) + 1);
System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
@@ -316,10 +249,8 @@ public final class DirectLogFetcher extends LogFetcher
buffer[position++] = 0;
}
protected final void sendBinlogDump(String fileName,
final long filePosition, final int serverId, boolean nonBlocking)
throws IOException
{
protected final void sendBinlogDump(String fileName, final long filePosition, final int serverId,
boolean nonBlocking) throws IOException {
position = NET_HEADER_SIZE;
putByte(COM_BINLOG_DUMP);
@@ -345,13 +276,10 @@ public final class DirectLogFetcher extends LogFetcher
*
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
*/
public boolean fetch() throws IOException
{
try
{
public boolean fetch() throws IOException {
try {
// Fetching packet header from input.
if (!fetch0(0, NET_HEADER_SIZE))
{
if (!fetch0(0, NET_HEADER_SIZE)) {
logger.warn("Reached end of input stream while fetching header");
return false;
}
@@ -359,61 +287,48 @@ public final class DirectLogFetcher extends LogFetcher
// Fetching the first packet(may a multi-packet).
int netlen = getUint24(PACKET_LEN_OFFSET);
int netnum = getUint8(PACKET_SEQ_OFFSET);
if (!fetch0(NET_HEADER_SIZE, netlen))
{
logger.warn("Reached end of input stream: packet #" + netnum
+ ", len = " + netlen);
if (!fetch0(NET_HEADER_SIZE, netlen)) {
logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
return false;
}
// Detecting error code.
final int mark = getUint8(NET_HEADER_SIZE);
if (mark != 0)
{
if (mark != 0) {
if (mark == 255) // error from master
{
// Indicates an error, for example trying to fetch from wrong
// Indicates an error, for example trying to fetch from
// wrong
// binlog position.
position = NET_HEADER_SIZE + 1;
final int errno = getInt16();
String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
String errmsg = getFixString(limit - position);
throw new IOException("Received error packet:"
+ " errno = " + errno + ", sqlstate = " + sqlstate
throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
+ " errmsg = " + errmsg);
}
else if (mark == 254)
{
} else if (mark == 254) {
// Indicates end of stream. It's not clear when this would
// be sent.
logger.warn("Received EOF packet from server, apparent"
+ " master disconnected.");
logger.warn("Received EOF packet from server, apparent" + " master disconnected.");
return false;
}
else
{
} else {
// Should not happen.
throw new IOException("Unexpected response " + mark
+ " while fetching binlog: packet #" + netnum
throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
+ ", len = " + netlen);
}
}
// The first packet is a multi-packet, concatenate the packets.
while (netlen == MAX_PACKET_LENGTH)
{
if (!fetch0(0, NET_HEADER_SIZE))
{
while (netlen == MAX_PACKET_LENGTH) {
if (!fetch0(0, NET_HEADER_SIZE)) {
logger.warn("Reached end of input stream while fetching header");
return false;
}
netlen = getUint24(PACKET_LEN_OFFSET);
netnum = getUint8(PACKET_SEQ_OFFSET);
if (!fetch0(limit, netlen))
{
logger.warn("Reached end of input stream: packet #"
+ netnum + ", len = " + netlen);
if (!fetch0(limit, netlen)) {
logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
return false;
}
}
@@ -423,43 +338,32 @@ public final class DirectLogFetcher extends LogFetcher
position = origin;
limit -= origin;
return true;
}
catch (SocketTimeoutException e)
{
} catch (SocketTimeoutException e) {
close(); /* Do cleanup */
logger.error("Socket timeout expired, closing connection", e);
throw e;
}
catch (InterruptedIOException e)
{
} catch (InterruptedIOException e) {
close(); /* Do cleanup */
logger.warn("I/O interrupted while reading from client socket", e);
throw e;
}
catch (IOException e)
{
} catch (IOException e) {
close(); /* Do cleanup */
logger.error("I/O error while reading from client socket", e);
throw e;
}
}
private final boolean fetch0(final int off, final int len)
throws IOException
{
private final boolean fetch0(final int off, final int len) throws IOException {
ensureCapacity(off + len);
for (int count, n = 0; n < len; n += count)
{
if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
{
for (int count, n = 0; n < len; n += count) {
if (0 > (count = mysqlInput.read(buffer, off + n, len - n))) {
// Reached end of input stream
return false;
}
}
if (limit < off + len)
limit = off + len;
if (limit < off + len) limit = off + len;
return true;
}
@@ -468,19 +372,14 @@ public final class DirectLogFetcher extends LogFetcher
*
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
*/
public void close() throws IOException
{
try
{
if (conn != null)
conn.close();
public void close() throws IOException {
try {
if (conn != null) conn.close();
conn = null;
mysqlInput = null;
mysqlOutput = null;
}
catch (SQLException e)
{
} catch (SQLException e) {
logger.warn("Unable to close connection", e);
}
}

1080
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java
View File

File diff suppressed because it is too large Load Diff

39
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java
View File

@@ -7,69 +7,56 @@ import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
/**
* TODO: Document Me!!
*
* NOTE: Log context will NOT write multi-threaded.
* TODO: Document Me!! NOTE: Log context will NOT write multi-threaded.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class LogContext
{
public final class LogContext {
private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
private FormatDescriptionLogEvent formatDescription;
private LogPosition logPosition;
public LogContext()
{
public LogContext(){
this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
}
public LogContext(FormatDescriptionLogEvent descriptionEvent)
{
public LogContext(FormatDescriptionLogEvent descriptionEvent){
this.formatDescription = descriptionEvent;
}
public final LogPosition getLogPosition()
{
public final LogPosition getLogPosition() {
return logPosition;
}
public final void setLogPosition(LogPosition logPosition)
{
public final void setLogPosition(LogPosition logPosition) {
this.logPosition = logPosition;
}
public final FormatDescriptionLogEvent getFormatDescription()
{
public final FormatDescriptionLogEvent getFormatDescription() {
return formatDescription;
}
public final void setFormatDescription(
FormatDescriptionLogEvent formatDescription)
{
public final void setFormatDescription(FormatDescriptionLogEvent formatDescription) {
this.formatDescription = formatDescription;
}
public final void putTable(TableMapLogEvent mapEvent)
{
public final void putTable(TableMapLogEvent mapEvent) {
mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
}
public final TableMapLogEvent getTable(final long tableId)
{
public final TableMapLogEvent getTable(final long tableId) {
return mapOfTable.get(Long.valueOf(tableId));
}
public final void clearAllTables()
{
public final void clearAllTables() {
mapOfTable.clear();
}
public void reset()
{
public void reset() {
formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
mapOfTable.clear();

249
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java
View File

@@ -61,28 +61,24 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class LogDecoder
{
public final class LogDecoder {
protected static final Log logger = LogFactory.getLog(LogDecoder.class);
protected final BitSet handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
public LogDecoder()
{
public LogDecoder(){
}
public LogDecoder(final int fromIndex, final int toIndex)
{
public LogDecoder(final int fromIndex, final int toIndex){
handleSet.set(fromIndex, toIndex);
}
public final void handle(final int fromIndex, final int toIndex)
{
public final void handle(final int fromIndex, final int toIndex) {
handleSet.set(fromIndex, toIndex);
}
public final void handle(final int flagIndex)
{
public final void handle(final int flagIndex) {
handleSet.set(flagIndex);
}
@@ -92,46 +88,30 @@ public final class LogDecoder
* @return <code>UknownLogEvent</code> if event type is unknown or skipped,
* <code>null</code> if buffer is not including a full event.
*/
public LogEvent decode(LogBuffer buffer, LogContext context)
throws IOException
{
public LogEvent decode(LogBuffer buffer, LogContext context) throws IOException {
final int limit = buffer.limit();
if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
{
LogHeader header = new LogHeader(buffer,
context.getFormatDescription());
if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) {
LogHeader header = new LogHeader(buffer, context.getFormatDescription());
final int len = header.getEventLen();
if (limit >= len)
{
if (limit >= len) {
LogEvent event;
/* Checking binary-log's header */
if (handleSet.get(header.getType()))
{
if (handleSet.get(header.getType())) {
buffer.limit(len);
try
{
try {
/* Decoding binary-log to event */
event = decode(buffer, header, context);
}
catch (IOException e)
{
if (logger.isWarnEnabled())
logger.warn("Decoding "
+ LogEvent.getTypeName(header.getType())
+ " failed from: "
+ context.getLogPosition(), e);
} catch (IOException e) {
if (logger.isWarnEnabled()) logger.warn("Decoding " + LogEvent.getTypeName(header.getType())
+ " failed from: " + context.getLogPosition(), e);
throw e;
}
finally
{
} finally {
buffer.limit(limit); /* Restore limit */
}
}
else
{
} else {
/* Ignore unsupported binary-log. */
event = new UnknownLogEvent(header);
}
@@ -152,9 +132,7 @@ public final class LogDecoder
*
* @return <code>UknownLogEvent</code> if event type is unknown or skipped.
*/
public static LogEvent decode(LogBuffer buffer, LogHeader header,
LogContext context) throws IOException
{
public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext context) throws IOException {
FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
LogPosition logPosition = context.getLogPosition();
@@ -171,234 +149,179 @@ public final class LogDecoder
buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
}
switch (header.getType())
{
case LogEvent.QUERY_EVENT:
{
QueryLogEvent event = new QueryLogEvent(header, buffer,
descriptionEvent);
switch (header.getType()) {
case LogEvent.QUERY_EVENT: {
QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.XID_EVENT:
{
XidLogEvent event = new XidLogEvent(header, buffer,
descriptionEvent);
case LogEvent.XID_EVENT: {
XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.TABLE_MAP_EVENT:
{
TableMapLogEvent mapEvent = new TableMapLogEvent(header,
buffer, descriptionEvent);
case LogEvent.TABLE_MAP_EVENT: {
TableMapLogEvent mapEvent = new TableMapLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
context.putTable(mapEvent);
return mapEvent;
}
case LogEvent.WRITE_ROWS_EVENT_V1:
{
RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
descriptionEvent);
case LogEvent.WRITE_ROWS_EVENT_V1: {
RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.UPDATE_ROWS_EVENT_V1:
{
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
descriptionEvent);
case LogEvent.UPDATE_ROWS_EVENT_V1: {
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.DELETE_ROWS_EVENT_V1:
{
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
descriptionEvent);
case LogEvent.DELETE_ROWS_EVENT_V1: {
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
event.fillTable(context);
return event;
}
case LogEvent.ROTATE_EVENT:
{
RotateLogEvent event = new RotateLogEvent(header, buffer,
descriptionEvent);
case LogEvent.ROTATE_EVENT: {
RotateLogEvent event = new RotateLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition = new LogPosition(event.getFilename(),
event.getPosition());
logPosition = new LogPosition(event.getFilename(), event.getPosition());
context.setLogPosition(logPosition);
return event;
}
case LogEvent.LOAD_EVENT:
case LogEvent.NEW_LOAD_EVENT:
{
LoadLogEvent event = new LoadLogEvent(header, buffer,
descriptionEvent);
case LogEvent.NEW_LOAD_EVENT: {
LoadLogEvent event = new LoadLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported SLAVE_EVENT from: "
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported SLAVE_EVENT from: "
+ context.getLogPosition());
break;
}
case LogEvent.CREATE_FILE_EVENT:
{
CreateFileLogEvent event = new CreateFileLogEvent(header,
buffer, descriptionEvent);
case LogEvent.CREATE_FILE_EVENT: {
CreateFileLogEvent event = new CreateFileLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.APPEND_BLOCK_EVENT:
{
AppendBlockLogEvent event = new AppendBlockLogEvent(header,
buffer, descriptionEvent);
case LogEvent.APPEND_BLOCK_EVENT: {
AppendBlockLogEvent event = new AppendBlockLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.DELETE_FILE_EVENT:
{
DeleteFileLogEvent event = new DeleteFileLogEvent(header,
buffer, descriptionEvent);
case LogEvent.DELETE_FILE_EVENT: {
DeleteFileLogEvent event = new DeleteFileLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.EXEC_LOAD_EVENT:
{
ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
buffer, descriptionEvent);
case LogEvent.EXEC_LOAD_EVENT: {
ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.START_EVENT_V3:
{
case LogEvent.START_EVENT_V3: {
/* This is sent only by MySQL <=4.x */
StartLogEventV3 event = new StartLogEventV3(header, buffer,
descriptionEvent);
StartLogEventV3 event = new StartLogEventV3(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.STOP_EVENT:
{
StopLogEvent event = new StopLogEvent(header, buffer,
descriptionEvent);
case LogEvent.STOP_EVENT: {
StopLogEvent event = new StopLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.INTVAR_EVENT:
{
IntvarLogEvent event = new IntvarLogEvent(header, buffer,
descriptionEvent);
case LogEvent.INTVAR_EVENT: {
IntvarLogEvent event = new IntvarLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.RAND_EVENT:
{
RandLogEvent event = new RandLogEvent(header, buffer,
descriptionEvent);
case LogEvent.RAND_EVENT: {
RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.USER_VAR_EVENT:
{
UserVarLogEvent event = new UserVarLogEvent(header, buffer,
descriptionEvent);
case LogEvent.USER_VAR_EVENT: {
UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.FORMAT_DESCRIPTION_EVENT:
{
descriptionEvent = new FormatDescriptionLogEvent(header,
buffer, descriptionEvent);
case LogEvent.FORMAT_DESCRIPTION_EVENT: {
descriptionEvent = new FormatDescriptionLogEvent(header, buffer, descriptionEvent);
context.setFormatDescription(descriptionEvent);
return descriptionEvent;
}
case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
case LogEvent.PRE_GA_WRITE_ROWS_EVENT: {
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Write_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: {
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Update_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
{
if (logger.isWarnEnabled())
logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
case LogEvent.PRE_GA_DELETE_ROWS_EVENT: {
if (logger.isWarnEnabled()) logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+ context.getLogPosition());
// ev = new Delete_rows_log_event_old(buf, event_len,
// description_event);
break;
}
case LogEvent.BEGIN_LOAD_QUERY_EVENT:
{
BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
header, buffer, descriptionEvent);
case LogEvent.BEGIN_LOAD_QUERY_EVENT: {
BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
{
ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
header, buffer, descriptionEvent);
case LogEvent.EXECUTE_LOAD_QUERY_EVENT: {
ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.INCIDENT_EVENT:
{
IncidentLogEvent event = new IncidentLogEvent(header, buffer,
descriptionEvent);
case LogEvent.INCIDENT_EVENT: {
IncidentLogEvent event = new IncidentLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.HEARTBEAT_LOG_EVENT:
{
HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
descriptionEvent);
case LogEvent.HEARTBEAT_LOG_EVENT: {
HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.IGNORABLE_LOG_EVENT:
{
case LogEvent.IGNORABLE_LOG_EVENT: {
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.ROWS_QUERY_LOG_EVENT:
{
case LogEvent.ROWS_QUERY_LOG_EVENT: {
RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
@@ -426,43 +349,37 @@ public final class LogDecoder
return event;
}
case LogEvent.GTID_LOG_EVENT:
case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
{
case LogEvent.ANONYMOUS_GTID_LOG_EVENT: {
GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
{
case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.ANNOTATE_ROWS_EVENT:
{
case LogEvent.ANNOTATE_ROWS_EVENT: {
AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.BINLOG_CHECKPOINT_EVENT:
{
case LogEvent.BINLOG_CHECKPOINT_EVENT: {
BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.GTID_EVENT:
{
case LogEvent.GTID_EVENT: {
MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
return event;
}
case LogEvent.GTID_LIST_EVENT:
{
case LogEvent.GTID_LIST_EVENT: {
MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
/* updating position in context */
logPosition.position = header.getLogPos();
@@ -470,8 +387,9 @@ public final class LogDecoder
}
default:
/*
Create an object of Ignorable_log_event for unrecognized sub-class.
So that SLAVE SQL THREAD will only update the position and continue.
* Create an object of Ignorable_log_event for unrecognized
* sub-class. So that SLAVE SQL THREAD will only update the
* position and continue.
*/
if ((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0) {
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
@@ -479,8 +397,7 @@ public final class LogDecoder
logPosition.position = header.getLogPos();
return event;
} else {
if (logger.isWarnEnabled())
logger.warn("Skipping unrecognized binlog event "
if (logger.isWarnEnabled()) logger.warn("Skipping unrecognized binlog event "
+ LogEvent.getTypeName(header.getType()) + " from: "
+ context.getLogPosition());
}

133
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java
View File

@@ -7,88 +7,66 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader;
/**
* Binary log event definitions. This includes generic code common to all types
* of log events, as well as specific code for each type of log event.
*
* - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
* in little endian, i.e., the least significant byte first, unless otherwise
* specified.
*
* representation of unsigned integers, called Packed Integer. A Packed Integer
* has the capacity of storing up to 8-byte integers, while small integers still
* can use 1, 3, or 4 bytes. The value of the first byte determines how to read
* the number, according to the following table:
*
* of log events, as well as specific code for each type of log event. - All
* numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored in
* little endian, i.e., the least significant byte first, unless otherwise
* specified. representation of unsigned integers, called Packed Integer. A
* Packed Integer has the capacity of storing up to 8-byte integers, while small
* integers still can use 1, 3, or 4 bytes. The value of the first byte
* determines how to read the number, according to the following table:
* <table>
* <caption>Format of Packed Integer</caption>
*
* <tr>
* <th>First byte</th>
* <th>Format</th>
* </tr>
*
* <tr>
* <td>0-250</td>
* <td>The first byte is the number (in the range 0-250), and no more bytes are
* used.</td>
* </tr>
*
* <tr>
* <td>252</td>
* <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
* </tr>
*
* <tr>
* <td>253</td>
* <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
* </tr>
*
* <tr>
* <td>254</td>
* <td>Eight more bytes are used. The number is in the range
* 0xffffff-0xffffffffffffffff.</td>
* </tr>
*
* </table>
*
* - Strings are stored in various formats. The format of each string is
* documented separately.
*
* @see mysql-5.1.60/sql/log_event.h
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public abstract class LogEvent
{
public abstract class LogEvent {
/*
* 3 is MySQL 4.x; 4 is MySQL 5.0.0.
* Compared to version 3, version 4 has:
* - a different Start_log_event, which includes info about the binary log
* 3 is MySQL 4.x; 4 is MySQL 5.0.0. Compared to version 3, version 4 has: -
* a different Start_log_event, which includes info about the binary log
* (sizes of headers); this info is included for better compatibility if the
* master's MySQL version is different from the slave's.
* - all events have a unique ID (the triplet (server_id, timestamp at server
* start, other) to be sure an event is not executed more than once in a
* multimaster setup, example:
* M1
* / \
* v v
* M2 M3
* \ /
* v v
* S
* if a query is run on M1, it will arrive twice on S, so we need that S
* remembers the last unique ID it has processed, to compare and know if the
* event should be skipped or not. Example of ID: we already have the server id
* (4 bytes), plus:
* master's MySQL version is different from the slave's. - all events have a
* unique ID (the triplet (server_id, timestamp at server start, other) to
* be sure an event is not executed more than once in a multimaster setup,
* example: M1 / \ v v M2 M3 \ / v v S if a query is run on M1, it will
* arrive twice on S, so we need that S remembers the last unique ID it has
* processed, to compare and know if the event should be skipped or not.
* Example of ID: we already have the server id (4 bytes), plus:
* timestamp_when_the_master_started (4 bytes), a counter (a sequence number
* which increments every time we write an event to the binlog) (3 bytes).
* Q: how do we handle when the counter is overflowed and restarts from 0 ?
*
* - Query and Load (Create or Execute) events may have a more precise
* timestamp (with microseconds), number of matched/affected/warnings rows
* and fields of session variables: SQL_MODE,
* FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
* charsets, the PASSWORD() version (old/new/...).
* and fields of session variables: SQL_MODE, FOREIGN_KEY_CHECKS,
* UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and charsets, the
* PASSWORD() version (old/new/...).
*/
public static final int BINLOG_VERSION = 4;
@@ -160,13 +138,15 @@ public abstract class LogEvent
public static final int INCIDENT_EVENT = 26;
/**
* Heartbeat event to be send by master at its idle time to ensure master's online status to slave
* Heartbeat event to be send by master at its idle time to ensure master's
* online status to slave
*/
public static final int HEARTBEAT_LOG_EVENT = 27;
/**
* In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
* case there is code for handling it, but which can be ignored if it is not recognized.
* In some situations, it is necessary to send over ignorable data to the
* slave: data that a slave can handle in case there is code for handling
* it, but which can be ignored if it is not recognized.
*/
public static final int IGNORABLE_LOG_EVENT = 28;
public static final int ROWS_QUERY_LOG_EVENT = 29;
@@ -189,23 +169,23 @@ public abstract class LogEvent
/* New Maria event numbers start from here */
public static final int ANNOTATE_ROWS_EVENT = 160;
/*
Binlog checkpoint event. Used for XA crash recovery on the master, not used
in replication.
A binlog checkpoint event specifies a binlog file such that XA crash
recovery can start from that file - and it is guaranteed to find all XIDs
that are prepared in storage engines but not yet committed.
* Binlog checkpoint event. Used for XA crash recovery on the master, not
* used in replication. A binlog checkpoint event specifies a binlog file
* such that XA crash recovery can start from that file - and it is
* guaranteed to find all XIDs that are prepared in storage engines but not
* yet committed.
*/
public static final int BINLOG_CHECKPOINT_EVENT = 161;
/*
Gtid event. For global transaction ID, used to start a new event group,
instead of the old BEGIN query event, and also to mark stand-alone
events.
* Gtid event. For global transaction ID, used to start a new event group,
* instead of the old BEGIN query event, and also to mark stand-alone
* events.
*/
public static final int GTID_EVENT = 162;
/*
Gtid list event. Logged at the start of every binlog, to record the
current replication state. This consists of the last GTID seen for
each replication domain.
* Gtid list event. Logged at the start of every binlog, to record the
* current replication state. This consists of the last GTID seen for each
* replication domain.
*/
public static final int GTID_LIST_EVENT = 163;
@@ -213,9 +193,8 @@ public abstract class LogEvent
public static final int ENUM_END_EVENT = 164;
/**
1 byte length, 1 byte format
Length is total length in bytes, including 2 byte header
Length values 0 and 1 are currently invalid and reserved.
* 1 byte length, 1 byte format Length is total length in bytes, including 2
* byte header Length values 0 and 1 are currently invalid and reserved.
*/
public static final int EXTRA_ROW_INFO_LEN_OFFSET = 0;
public static final int EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
@@ -260,11 +239,10 @@ public abstract class LogEvent
public static final int MARIA_SLAVE_CAPABILITY_MINE = MARIA_SLAVE_CAPABILITY_GTID;
/**
For an event, 'e', carrying a type code, that a slave,
's', does not recognize, 's' will check 'e' for
LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
is ignored. Otherwise, 's' acknowledges that it has
found an unknown event in the relay log.
* For an event, 'e', carrying a type code, that a slave, 's', does not
* recognize, 's' will check 'e' for LOG_EVENT_IGNORABLE_F, and if the flag
* is set, then 'e' is ignored. Otherwise, 's' acknowledges that it has
* found an unknown event in the relay log.
*/
public static final int LOG_EVENT_IGNORABLE_F = 0x80;
@@ -300,10 +278,8 @@ public abstract class LogEvent
public static final int MYSQL_TYPE_STRING = 254;
public static final int MYSQL_TYPE_GEOMETRY = 255;
public static String getTypeName(final int type)
{
switch (type)
{
public static String getTypeName(final int type) {
switch (type) {
case START_EVENT_V3:
return "Start_v3";
case STOP_EVENT:
@@ -383,16 +359,14 @@ public abstract class LogEvent
protected final LogHeader header;
protected LogEvent(LogHeader header)
{
protected LogEvent(LogHeader header){
this.header = header;
}
/**
* Return event header.
*/
public final LogHeader getHeader()
{
public final LogHeader getHeader() {
return header;
}
@@ -400,16 +374,14 @@ public abstract class LogEvent
* The total size of this event, in bytes. In other words, this is the sum
* of the sizes of Common-Header, Post-Header, and Body.
*/
public final int getEventLen()
{
public final int getEventLen() {
return header.getEventLen();
}
/**
* Server ID of the server that created the event.
*/
public final long getServerId()
{
public final long getServerId() {
return header.getServerId();
}
@@ -420,18 +392,15 @@ public abstract class LogEvent
* file. In a relay log, this is the position of the next event in the
* master's binlog.
*/
public final long getLogPos()
{
public final long getLogPos() {
return header.getLogPos();
}
/**
* The time when the query started, in seconds since 1970.
*/
public final long getWhen()
{
public final long getWhen() {
return header.getWhen();
}
}

22
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java
View File

@@ -29,8 +29,8 @@ import java.util.Arrays;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public abstract class LogFetcher extends LogBuffer implements Closeable
{
public abstract class LogFetcher extends LogBuffer implements Closeable {
/** Default initial capacity. */
public static final int DEFAULT_INITIAL_CAPACITY = 8192;
@@ -42,18 +42,15 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
protected final float factor;
public LogFetcher()
{
public LogFetcher(){
this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
}
public LogFetcher(final int initialCapacity)
{
public LogFetcher(final int initialCapacity){
this(initialCapacity, DEFAULT_GROWTH_FACTOR);
}
public LogFetcher(final int initialCapacity, final float growthFactor)
{
public LogFetcher(final int initialCapacity, final float growthFactor){
this.buffer = new byte[initialCapacity];
this.factor = growthFactor;
}
@@ -65,15 +62,12 @@ public abstract class LogFetcher extends LogBuffer implements Closeable
*
* @param minCapacity the desired minimum capacity
*/
protected final void ensureCapacity(final int minCapacity)
{
protected final void ensureCapacity(final int minCapacity) {
final int oldCapacity = buffer.length;
if (minCapacity > oldCapacity)
{
if (minCapacity > oldCapacity) {
int newCapacity = (int) (oldCapacity * factor);
if (newCapacity < minCapacity)
newCapacity = minCapacity;
if (newCapacity < minCapacity) newCapacity = minCapacity;
buffer = Arrays.copyOf(buffer, newCapacity);
}

53
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java
View File

@@ -6,8 +6,8 @@ package com.taobao.tddl.dbsync.binlog;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class LogPosition implements Cloneable, Comparable<LogPosition>
{
public class LogPosition implements Cloneable, Comparable<LogPosition> {
/* binlog file's name */
protected String fileName;
@@ -19,8 +19,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @param fileName file name for binlog files: mysql-bin.000001
*/
public LogPosition(String fileName)
{
public LogPosition(String fileName){
this.fileName = fileName;
this.position = 0L;
}
@@ -30,8 +29,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @param fileName file name for binlog files: mysql-bin.000001
*/
public LogPosition(String fileName, final long position)
{
public LogPosition(String fileName, final long position){
this.fileName = fileName;
this.position = position;
}
@@ -39,31 +37,24 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
/**
* Binlog position copy init.
*/
public LogPosition(LogPosition source)
{
public LogPosition(LogPosition source){
this.fileName = source.fileName;
this.position = source.position;
}
public final String getFileName()
{
public final String getFileName() {
return fileName;
}
public final long getPosition()
{
public final long getPosition() {
return position;
}
/* Clone binlog position without CloneNotSupportedException */
public LogPosition clone()
{
try
{
public LogPosition clone() {
try {
return (LogPosition) super.clone();
}
catch (CloneNotSupportedException e)
{
} catch (CloneNotSupportedException e) {
// Never happend
return null;
}
@@ -72,12 +63,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
/**
* Compares with the specified fileName and position.
*/
public final int compareTo(String fileName, final long position)
{
public final int compareTo(String fileName, final long position) {
final int val = this.fileName.compareTo(fileName);
if (val == 0)
{
if (val == 0) {
return (int) (this.position - position);
}
return val;
@@ -88,12 +77,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/
public int compareTo(LogPosition o)
{
public int compareTo(LogPosition o) {
final int val = fileName.compareTo(o.fileName);
if (val == 0)
{
if (val == 0) {
return (int) (position - o.position);
}
return val;
@@ -104,13 +91,10 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @see java.lang.Object#equals(java.lang.Object)
*/
public boolean equals(Object obj)
{
if (obj instanceof LogPosition)
{
public boolean equals(Object obj) {
if (obj instanceof LogPosition) {
LogPosition pos = ((LogPosition) obj);
return fileName.equals(pos.fileName)
&& (this.position == pos.position);
return fileName.equals(pos.fileName) && (this.position == pos.position);
}
return false;
}
@@ -120,8 +104,7 @@ public class LogPosition implements Cloneable, Comparable<LogPosition>
*
* @see java.lang.Object#toString()
*/
public String toString()
{
public String toString() {
return fileName + ':' + position;
}
}

17
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java
View File

@@ -9,8 +9,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class AppendBlockLogEvent extends LogEvent
{
public class AppendBlockLogEvent extends LogEvent {
private final LogBuffer blockBuf;
private final int blockLen;
@@ -19,9 +19,7 @@ public class AppendBlockLogEvent extends LogEvent
/* AB = "Append Block" */
public static final int AB_FILE_ID_OFFSET = 0;
public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -36,18 +34,15 @@ public class AppendBlockLogEvent extends LogEvent
blockBuf = buffer.duplicate(blockLen);
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
public final LogBuffer getBuffer()
{
public final LogBuffer getBuffer() {
return blockBuf;
}
public final byte[] getData()
{
public final byte[] getData() {
return blockBuf.getData();
}
}

8
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java
View File

@@ -10,11 +10,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
{
public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent {
public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

45
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java
View File

@@ -8,8 +8,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class CreateFileLogEvent extends LoadLogEvent
{
public final class CreateFileLogEvent extends LoadLogEvent {
protected LogBuffer blockBuf;
protected int blockLen;
protected long fileId;
@@ -20,9 +20,7 @@ public final class CreateFileLogEvent extends LoadLogEvent
public static final int CF_FILE_ID_OFFSET = 0;
public static final int CF_DATA_OFFSET = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public CreateFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
final int headerLen = descriptionEvent.commonHeaderLen;
@@ -30,44 +28,37 @@ public final class CreateFileLogEvent extends LoadLogEvent
final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
copyLogEvent(buffer,
((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
: (headerLen + loadHeaderLen + createFileHeaderLen)),
((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen) : (headerLen + loadHeaderLen + createFileHeaderLen)),
descriptionEvent);
if (descriptionEvent.binlogVersion != 1)
{
fileId = buffer.getUint32(headerLen + loadHeaderLen
+ CF_FILE_ID_OFFSET);
if (descriptionEvent.binlogVersion != 1) {
fileId = buffer.getUint32(headerLen + loadHeaderLen + CF_FILE_ID_OFFSET);
/*
Note that it's ok to use get_data_size() below, because it is computed
with values we have already read from this event (because we called
copy_log_event()); we are not using slave's format info to decode
master's format, we are really using master's format info.
Anyway, both formats should be identical (except the common_header_len)
as these Load events are not changed between 4.0 and 5.0 (as logging of
LOAD DATA INFILE does not use Load_log_event in 5.0).
* Note that it's ok to use get_data_size() below, because it is
* computed with values we have already read from this event
* (because we called copy_log_event()); we are not using slave's
* format info to decode master's format, we are really using
* master's format info. Anyway, both formats should be identical
* (except the common_header_len) as these Load events are not
* changed between 4.0 and 5.0 (as logging of LOAD DATA INFILE does
* not use Load_log_event in 5.0).
*/
blockLen = buffer.limit() - buffer.position();
blockBuf = buffer.duplicate(blockLen);
}
else
{
} else {
initedFromOld = true;
}
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
public final LogBuffer getBuffer()
{
public final LogBuffer getBuffer() {
return blockBuf;
}
public final byte[] getData()
{
public final byte[] getData() {
return blockBuf.getData();
}
}

11
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java
View File

@@ -9,16 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class DeleteFileLogEvent extends LogEvent
{
public final class DeleteFileLogEvent extends LogEvent {
private final long fileId;
/* DF = "Delete File" */
public static final int DF_FILE_ID_OFFSET = 0;
public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -26,8 +24,7 @@ public final class DeleteFileLogEvent extends LogEvent
fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
}

8
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java
View File

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class DeleteRowsLogEvent extends RowsLogEvent
{
public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class DeleteRowsLogEvent extends RowsLogEvent {
public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

11
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java
View File

@@ -9,16 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class ExecuteLoadLogEvent extends LogEvent
{
public final class ExecuteLoadLogEvent extends LogEvent {
private final long fileId;
/* EL = "Execute Load" */
public static final int EL_FILE_ID_OFFSET = 0;
public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -26,8 +24,7 @@ public final class ExecuteLoadLogEvent extends LogEvent
fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
}

35
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java
View File

@@ -8,7 +8,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* Event responsible for LOAD DATA execution, it similar to Query_log_event but
* before executing the query it substitutes original filename in LOAD DATA
* query with name of temporary file.
*
* <ul>
* <li>4 bytes. The ID of the file to load.</li>
* <li>4 bytes. The start position within the statement for filename
@@ -22,8 +21,8 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
{
public final class ExecuteLoadQueryLogEvent extends QueryLogEvent {
/** file_id of temporary file */
private long fileId;
@@ -53,9 +52,8 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
public static final int ELQ_FN_POS_END_OFFSET = ELQ_FILE_ID_OFFSET + 8;
public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent) throws IOException
{
public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
super(header, buffer, descriptionEvent);
buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
@@ -66,36 +64,27 @@ public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
final int len = query.length();
if (fnPosStart > len || fnPosEnd > len
|| dupHandling > LOAD_DUP_REPLACE)
{
throw new IOException(String.format(
"Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
+ "fn_pos_end=%d, dup_handling=%d", fnPosStart,
fnPosEnd, dupHandling));
if (fnPosStart > len || fnPosEnd > len || dupHandling > LOAD_DUP_REPLACE) {
throw new IOException(String.format("Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
+ "fn_pos_end=%d, dup_handling=%d", fnPosStart, fnPosEnd, dupHandling));
}
}
public final int getFilenamePosStart()
{
public final int getFilenamePosStart() {
return fnPosStart;
}
public final int getFilenamePosEnd()
{
public final int getFilenamePosEnd() {
return fnPosEnd;
}
public final String getFilename()
{
if (query != null)
return query.substring(fnPosStart, fnPosEnd).trim();
public final String getFilename() {
if (query != null) return query.substring(fnPosStart, fnPosEnd).trim();
return null;
}
public final long getFileId()
{
public final long getFileId() {
return fileId;
}
}

65
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java
View File

@@ -5,8 +5,8 @@ import java.io.IOException;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
/**
* For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
* ordinary events).
* For binlog version 4. This event is saved by threads which read it, as they
* need it for future use (to decode the ordinary events).
*
* @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
@@ -15,8 +15,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
public final class FormatDescriptionLogEvent extends StartLogEventV3 {
/**
* The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
* exist in binlogs, it does not have a format).
* The number of types we handle in Format_description_log_event
* (UNKNOWN_EVENT is not to be handled, it does not exist in binlogs, it
* does not have a format).
*/
public static final int LOG_EVENT_TYPES = (ENUM_END_EVENT - 1);
@@ -70,8 +71,9 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
public static final long checksumVersionProduct = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
* 256 + checksumVersionSplit[2];
/**
* The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
* LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
* The size of the fixed header which _all_ events have (for binlogs written
* by this version, this is equal to LOG_EVENT_HEADER_LEN), except
* FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
* LOG_EVENT_MINIMAL_HEADER_LEN).
*/
protected final int commonHeaderLen;
@@ -105,7 +107,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
calcServerVersionSplit();
long calc = getVersionProduct();
if (calc >= checksumVersionProduct) {
/* the last bytes are the checksum alg desc and value (or value's room) */
/*
* the last bytes are the checksum alg desc and value (or value's
* room)
*/
numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
}
@@ -147,7 +152,10 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
commonHeaderLen = LOG_EVENT_HEADER_LEN;
numberOfEventTypes = LOG_EVENT_TYPES;
/* Note: all event types must explicitly fill in their lengths here. */
/*
* Note: all event types must explicitly fill in their lengths
* here.
*/
postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
@@ -171,11 +179,14 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
/*
* We here have the possibility to simulate a master of before we changed the table map id to be stored
* in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
* compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
* that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
* test (rpl_row_4_bytes) too.
* We here have the possibility to simulate a master of before
* we changed the table map id to be stored in 6 bytes: when it
* was stored in 4 bytes (=> post_header_len was 6). This is
* used to test backward compatibility. This code can be removed
* after a few months (today is Dec 21st 2005), when we know
* that the 4-byte masters are not deployed anymore (check with
* Tomas Ulin first!), and the accompanying test
* (rpl_row_4_bytes) too.
*/
postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
@@ -195,17 +206,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
case 3: /* 4.0.x x>=2 */
/*
* We build an artificial (i.e. not sent by the master) event, which describes what those old master
* versions send.
* We build an artificial (i.e. not sent by the master) event,
* which describes what those old master versions send.
*/
serverVersion = "4.0";
commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
/*
* The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
* in older versions. We use the events known by version 3, even if version 1 had only a subset of them
* (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
* detect less corruptions).
* The first new event in binlog version 4 is Format_desc. So
* any event type after that does not exist in older versions.
* We use the events known by version 3, even if version 1 had
* only a subset of them (this is not a problem: it uses a few
* bytes for nothing but unifies code; it does not make the
* slave detect less corruptions).
*/
numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
@@ -222,17 +235,19 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
case 1: /* 3.23 */
/*
* We build an artificial (i.e. not sent by the master) event, which describes what those old master
* versions send.
* We build an artificial (i.e. not sent by the master) event,
* which describes what those old master versions send.
*/
serverVersion = "3.23";
commonHeaderLen = OLD_HEADER_LEN;
/*
* The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
* in older versions. We use the events known by version 3, even if version 1 had only a subset of them
* (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
* detect less corruptions).
* The first new event in binlog version 4 is Format_desc. So
* any event type after that does not exist in older versions.
* We use the events known by version 3, even if version 1 had
* only a subset of them (this is not a problem: it uses a few
* bytes for nothing but unifies code; it does not make the
* slave detect less corruptions).
*/
numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;

26
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java
View File

@@ -6,19 +6,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* <pre>
* Base class for ignorable log events. Events deriving from
this class can be safely ignored by slaves that cannot
recognize them. Newer slaves, will be able to read and
handle them. This has been designed to be an open-ended
architecture, so adding new derived events shall not harm
the old slaves that support ignorable log event mechanism
(they will just ignore unrecognized ignorable events).
@note The only thing that makes an event ignorable is that it has
the LOG_EVENT_IGNORABLE_F flag set. It is not strictly necessary
that ignorable event types derive from Ignorable_log_event; they may
just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
argument to the Log_event constructor.
</pre>
* this class can be safely ignored by slaves that cannot
* recognize them. Newer slaves, will be able to read and
* handle them. This has been designed to be an open-ended
* architecture, so adding new derived events shall not harm
* the old slaves that support ignorable log event mechanism
* (they will just ignore unrecognized ignorable events).
*
* @note The only thing that makes an event ignorable is that it has
* the LOG_EVENT_IGNORABLE_F flag set. It is not strictly necessary
* that ignorable event types derive from Ignorable_log_event; they may
* just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
* argument to the Log_event constructor.
* </pre>
*
* @author jianghang 2013-4-8 上午12:36:29
* @version 1.0.3

26
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java
View File

@@ -5,12 +5,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* Class representing an incident, an occurance out of the ordinary, that
* happened on the master.
*
* The event is used to inform the slave that something out of the ordinary
* happened on the master that might cause the database to be in an inconsistent
* state.
*
* happened on the master. The event is used to inform the slave that something
* out of the ordinary happened on the master that might cause the database to
* be in an inconsistent state.
* <table id="IncidentFormat">
* <caption>Incident event format</caption>
* <tr>
@@ -38,8 +35,8 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class IncidentLogEvent extends LogEvent
{
public final class IncidentLogEvent extends LogEvent {
public static final int INCIDENT_NONE = 0;
/** There are possibly lost events in the replication stream */
@@ -51,9 +48,7 @@ public final class IncidentLogEvent extends LogEvent
private final int incident;
private final String message;
public IncidentLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public IncidentLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -61,8 +56,7 @@ public final class IncidentLogEvent extends LogEvent
buffer.position(commonHeaderLen);
final int incidentNumber = buffer.getUint16();
if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
{
if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE) {
// If the incident is not recognized, this binlog event is
// invalid. If we set incident_number to INCIDENT_NONE, the
// invalidity will be detected by is_valid().
@@ -76,13 +70,11 @@ public final class IncidentLogEvent extends LogEvent
message = buffer.getString();
}
public final int getIncident()
{
public final int getIncident() {
return incident;
}
public final String getMessage()
{
public final String getMessage() {
return message;
}
}

35
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java
View File

@@ -6,54 +6,43 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* An Intvar_log_event will be created just before a Query_log_event, if the
* query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
* Intvar_log_event holds the value of one of these variables.
*
* Binary Format
*
* The Post-Header for this event type is empty. The Body has two components:
*
* Intvar_log_event holds the value of one of these variables. Binary Format The
* Post-Header for this event type is empty. The Body has two components:
* <table>
* <caption>Body for Intvar_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>type</td>
* <td>1 byte enumeration</td>
* <td>One byte identifying the type of variable stored. Currently, two
* identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
* </tr>
*
* <tr>
* <td>value</td>
* <td>8 byte unsigned integer</td>
* <td>The value of the variable.</td>
* </tr>
*
* </table>
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class IntvarLogEvent extends LogEvent
{
public final class IntvarLogEvent extends LogEvent {
/**
* Fixed data part: Empty
*
* <p>
* Variable data part:
*
* <ul>
* <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
* 1 or INSERT_ID_EVENT = 2.</li>
* <li>8 bytes. An unsigned integer indicating the value to be used for the
* LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final long value;
@@ -68,31 +57,25 @@ public final class IntvarLogEvent extends LogEvent
public static final int LAST_INSERT_ID_EVENT = 1;
public static final int INSERT_ID_EVENT = 2;
public IntvarLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public IntvarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
/* The Post-Header is empty. The Varible Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+ I_TYPE_OFFSET);
type = buffer.getInt8(); // I_TYPE_OFFSET
value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
}
public final int getType()
{
public final int getType() {
return type;
}
public final long getValue()
{
public final long getValue() {
return value;
}
public final String getQuery()
{
public final String getQuery() {
return "SET INSERT_ID = " + value;
}
}

129
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java
View File

@@ -26,16 +26,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* </pre>
*
* Binary Format: The Post-Header consists of the following six components.
*
* <table>
* <caption>Post-Header for Load_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>slave_proxy_id</td>
* <td>4 byte unsigned integer</td>
@@ -45,71 +42,57 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* temporary table local to the client. The slave_proxy_id is used to
* distinguish temporary tables that belong to different clients.</td>
* </tr>
*
* <tr>
* <td>exec_time</td>
* <td>4 byte unsigned integer</td>
* <td>The time from when the query started to when it was logged in the binlog,
* in seconds.</td>
* </tr>
*
* <tr>
* <td>skip_lines</td>
* <td>4 byte unsigned integer</td>
* <td>The number on line (14) above, if present, or 0 if line (14) is left out.
* </td>
* </tr>
*
* <tr>
* <td>table_name_len</td>
* <td>1 byte unsigned integer</td>
* <td>The length of 'table_name' on line (4) above.</td>
* </tr>
*
* <tr>
* <td>db_len</td>
* <td>1 byte unsigned integer</td>
* <td>The length of 'db' on line (1) above.</td>
* </tr>
*
* <tr>
* <td>num_fields</td>
* <td>4 byte unsigned integer</td>
* <td>The number n of fields on line (15) above.</td>
* </tr>
* </table>
*
* The Body contains the following components.
*
* <table>
* <caption>Body of Load_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>sql_ex</td>
* <td>variable length</td>
*
* <td>Describes the part of the query on lines (3) and (5)&ndash;(13) above.
* More precisely, it stores the five strings (on lines) field_term (6),
* enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
* bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
* OPTIONALLY (7).
*
* The data is stored in one of two formats, called "old" and "new". The type
* field of Common-Header determines which of these two formats is used: type
* LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
* that the new format is used. When MySQL writes a Load_log_event, it uses the
* new format if at least one of the five strings is two or more bytes long.
* Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
* used.
*
* The new and old format differ in the way the five strings are stored.
*
* OPTIONALLY (7). The data is stored in one of two formats, called "old" and
* "new". The type field of Common-Header determines which of these two formats
* is used: type LOAD_EVENT means that the old format is used, and type
* NEW_LOAD_EVENT means that the new format is used. When MySQL writes a
* Load_log_event, it uses the new format if at least one of the five strings is
* two or more bytes long. Otherwise (i.e., if all strings are 0 or 1 bytes
* long), the old format is used. The new and old format differ in the way the
* five strings are stored.
* <ul>
* <li>In the new format, the strings are stored in the order field_term,
* enclosed, escaped, line_term, line_start. Each string consists of a length (1
@@ -117,7 +100,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
* IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
* the presence of the corresponding keyword in the SQL query.
*
* <li>In the old format, we know that each string has length 0 or 1. Therefore,
* only the first byte of each string is stored. The order of the strings is the
* same as in the new format. These five bytes are followed by the same 1 byte
@@ -128,18 +110,15 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
* LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
* </ul>
*
* Thus, the size of the new format is 6 bytes + the sum of the sizes of the
* five strings. The size of the old format is always 7 bytes.</td>
* </tr>
*
* <tr>
* <td>field_lens</td>
* <td>num_fields 1 byte unsigned integers</td>
* <td>An array of num_fields integers representing the length of each field in
* the query. (num_fields is from the Post-Header).</td>
* </tr>
*
* <tr>
* <td>fields</td>
* <td>num_fields null-terminated strings</td>
@@ -148,7 +127,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* the num_fields array.) The total length of all strings equals to the sum of
* all field_lens, plus num_fields bytes for all the trailing zeros.</td>
* </tr>
*
* <tr>
* <td>table_name</td>
* <td>null-terminated string of length table_len+1 bytes</td>
@@ -156,31 +134,27 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* trailing zero is actually redundant since the table_len is known from
* Post-Header.)</td>
* </tr>
*
* <tr>
* <td>db</td>
* <td>null-terminated string of length db_len+1 bytes</td>
* <td>The 'db' from the query, as a null-terminated string. (The trailing zero
* is actually redundant since the db_len is known from Post-Header.)</td>
* </tr>
*
* <tr>
* <td>file_name</td>
* <td>variable length string without trailing zero, extending to the end of the
* event (determined by the length field of the Common-Header)</td>
* <td>The 'file_name' from the query.</td>
* </tr>
*
* </table>
*
* This event type is understood by current versions, but only generated by
* MySQL 3.23 and earlier.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class LoadLogEvent extends LogEvent
{
public class LoadLogEvent extends LogEvent {
private String table;
private String db;
private String fname;
@@ -210,10 +184,9 @@ public class LoadLogEvent extends LogEvent
public static final int L_DATA_OFFSET = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
/*
These are flags and structs to handle all the LOAD DATA INFILE options (LINES
TERMINATED etc).
DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
DATA).
* These are flags and structs to handle all the LOAD DATA INFILE options
* (LINES TERMINATED etc). DUMPFILE_FLAG is probably useless (DUMPFILE is a
* clause of SELECT, not of LOAD DATA).
*/
public static final int DUMPFILE_FLAG = 0x1;
public static final int OPT_ENCLOSED_FLAG = 0x2;
@@ -226,9 +199,7 @@ public class LoadLogEvent extends LogEvent
public static final int LINE_START_EMPTY = 0x8;
public static final int ESCAPED_EMPTY = 0x10;
public LoadLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public LoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
@@ -236,8 +207,8 @@ public class LoadLogEvent extends LogEvent
* I (Guilhem) manually tested replication of LOAD DATA INFILE for
* 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
*/
copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
+ descriptionEvent.commonHeaderLen : loadHeaderLen
copyLogEvent(buffer,
((header.type == LOAD_EVENT) ? loadHeaderLen + descriptionEvent.commonHeaderLen : loadHeaderLen
+ FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
descriptionEvent);
}
@@ -245,9 +216,7 @@ public class LoadLogEvent extends LogEvent
/**
* @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
*/
protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
FormatDescriptionLogEvent descriptionEvent)
{
protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset, FormatDescriptionLogEvent descriptionEvent) {
/* this is the beginning of the post-header */
buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
@@ -262,8 +231,7 @@ public class LoadLogEvent extends LogEvent
* Sql_ex.init() on success returns the pointer to the first byte after
* the sql_ex structure, which is the start of field lengths array.
*/
if (header.type != LOAD_EVENT /* use_new_format */)
{
if (header.type != LOAD_EVENT /* use_new_format */) {
/*
* The code below assumes that buf will not disappear from under our
* feet during the lifetime of the event. This assumption holds true
@@ -279,9 +247,7 @@ public class LoadLogEvent extends LogEvent
escaped = buffer.getString();
optFlags = buffer.getInt8();
emptyFlags = 0;
}
else
{
} else {
fieldTerm = buffer.getFixString(1);
enclosed = buffer.getFixString(1);
lineTerm = buffer.getFixString(1);
@@ -290,23 +256,17 @@ public class LoadLogEvent extends LogEvent
optFlags = buffer.getUint8();
emptyFlags = buffer.getUint8();
if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
fieldTerm = null;
if ((emptyFlags & ENCLOSED_EMPTY) != 0)
enclosed = null;
if ((emptyFlags & LINE_TERM_EMPTY) != 0)
lineTerm = null;
if ((emptyFlags & LINE_START_EMPTY) != 0)
lineStart = null;
if ((emptyFlags & ESCAPED_EMPTY) != 0)
escaped = null;
if ((emptyFlags & FIELD_TERM_EMPTY) != 0) fieldTerm = null;
if ((emptyFlags & ENCLOSED_EMPTY) != 0) enclosed = null;
if ((emptyFlags & LINE_TERM_EMPTY) != 0) lineTerm = null;
if ((emptyFlags & LINE_START_EMPTY) != 0) lineStart = null;
if ((emptyFlags & ESCAPED_EMPTY) != 0) escaped = null;
}
final int fieldLenPos = buffer.position();
buffer.forward(numFields);
fields = new String[numFields];
for (int i = 0; i < numFields; i++)
{
for (int i = 0; i < numFields; i++) {
final int fieldLen = buffer.getUint8(fieldLenPos + i);
fields[i] = buffer.getFixString(fieldLen + 1);
}
@@ -324,68 +284,55 @@ public class LoadLogEvent extends LogEvent
buffer.forward(1); // The + 1 is for \0 terminating fname
}
public final String getTable()
{
public final String getTable() {
return table;
}
public final String getDb()
{
public final String getDb() {
return db;
}
public final String getFname()
{
public final String getFname() {
return fname;
}
public final int getSkipLines()
{
public final int getSkipLines() {
return skipLines;
}
public final String[] getFields()
{
public final String[] getFields() {
return fields;
}
public final String getFieldTerm()
{
public final String getFieldTerm() {
return fieldTerm;
}
public final String getLineTerm()
{
public final String getLineTerm() {
return lineTerm;
}
public final String getLineStart()
{
public final String getLineStart() {
return lineStart;
}
public final String getEnclosed()
{
public final String getEnclosed() {
return enclosed;
}
public final String getEscaped()
{
public final String getEscaped() {
return escaped;
}
public final int getOptFlags()
{
public final int getOptFlags() {
return optFlags;
}
public final int getEmptyFlags()
{
public final int getEmptyFlags() {
return emptyFlags;
}
public final long getExecTime()
{
public final long getExecTime() {
return execTime;
}
}

150
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java
View File

@@ -11,41 +11,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* different format and length even for different events of the same type. The
* binary formats of Post-Header and Body are documented separately in each
* subclass. The binary format of Common-Header is as follows.
*
* <table>
* <caption>Common-Header</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>timestamp</td>
* <td>4 byte unsigned integer</td>
* <td>The time when the query started, in seconds since 1970.</td>
* </tr>
*
* <tr>
* <td>type</td>
* <td>1 byte enumeration</td>
* <td>See enum #Log_event_type.</td>
* </tr>
*
* <tr>
* <td>server_id</td>
* <td>4 byte unsigned integer</td>
* <td>Server ID of the server that created the event.</td>
* </tr>
*
* <tr>
* <td>total_size</td>
* <td>4 byte unsigned integer</td>
* <td>The total size of this event, in bytes. In other words, this is the sum
* of the sizes of Common-Header, Post-Header, and Body.</td>
* </tr>
*
* <tr>
* <td>master_position</td>
* <td>4 byte unsigned integer</td>
@@ -54,24 +47,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* the position of the next event, in bytes from the beginning of the file. In a
* relay log, this is the position of the next event in the master's binlog.</td>
* </tr>
*
* <tr>
* <td>flags</td>
* <td>2 byte bitfield</td>
* <td>See Log_event::flags.</td>
* </tr>
* </table>
*
* Summing up the numbers above, we see that the total size of the common header
* is 19 bytes.
*
* @see mysql-5.1.60/sql/log_event.cc
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class LogHeader
{
public final class LogHeader {
protected final int type;
/**
@@ -114,34 +104,29 @@ public final class LogHeader
/**
* The value is set by caller of FD constructor and
* Log_event::write_header() for the rest.
* In the FD case it's propagated into the last byte
* of post_header_len[] at FD::write().
* On the slave side the value is assigned from post_header_len[last]
* of the last seen FD event.
* Log_event::write_header() for the rest. In the FD case it's propagated
* into the last byte of post_header_len[] at FD::write(). On the slave side
* the value is assigned from post_header_len[last] of the last seen FD
* event.
*/
protected int checksumAlg;
/**
Placeholder for event checksum while writing to binlog.
* Placeholder for event checksum while writing to binlog.
*/
protected long crc; // ha_checksum
/* for Start_event_v3 */
public LogHeader(final int type)
{
public LogHeader(final int type){
this.type = type;
}
public LogHeader(LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public LogHeader(LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
when = buffer.getUint32();
type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET;
serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET;
eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET;
if (descriptionEvent.binlogVersion == 1)
{
if (descriptionEvent.binlogVersion == 1) {
logPos = 0;
flags = 0;
return;
@@ -150,48 +135,43 @@ public final class LogHeader
/* 4.0 or newer */
logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET
/*
If the log is 4.0 (so here it can only be a 4.0 relay log read by
the SQL thread or a 4.0 master binlog read by the I/O thread),
log_pos is the beginning of the event: we transform it into the end
of the event, which is more useful.
But how do you know that the log is 4.0: you know it if
description_event is version 3 *and* you are not reading a
Format_desc (remember that mysqlbinlog starts by assuming that 5.0
logs are in 4.0 format, until it finds a Format_desc).
* If the log is 4.0 (so here it can only be a 4.0 relay log read by the
* SQL thread or a 4.0 master binlog read by the I/O thread), log_pos is
* the beginning of the event: we transform it into the end of the
* event, which is more useful. But how do you know that the log is 4.0:
* you know it if description_event is version 3 *and* you are not
* reading a Format_desc (remember that mysqlbinlog starts by assuming
* that 5.0 logs are in 4.0 format, until it finds a Format_desc).
*/
if (descriptionEvent.binlogVersion == 3
&& type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0)
{
if (descriptionEvent.binlogVersion == 3 && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0) {
/*
If log_pos=0, don't change it. log_pos==0 is a marker to mean
"don't change rli->group_master_log_pos" (see
inc_group_relay_log_pos()). As it is unreal log_pos, adding the
event len's is nonsense. For example, a fake Rotate event should
not have its log_pos (which is 0) changed or it will modify
Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
value of (a non-zero offset which does not exist in the master's
binlog, so which will cause problems if the user uses this value
in CHANGE MASTER).
* If log_pos=0, don't change it. log_pos==0 is a marker to mean
* "don't change rli->group_master_log_pos" (see
* inc_group_relay_log_pos()). As it is unreal log_pos, adding the
* event len's is nonsense. For example, a fake Rotate event should
* not have its log_pos (which is 0) changed or it will modify
* Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
* value of (a non-zero offset which does not exist in the master's
* binlog, so which will cause problems if the user uses this value
* in CHANGE MASTER).
*/
logPos += eventLen; /* purecov: inspected */
}
flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET
if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT)
|| (type == LogEvent.ROTATE_EVENT))
{
if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT) || (type == LogEvent.ROTATE_EVENT)) {
/*
These events always have a header which stops here (i.e. their
header is FROZEN).
* These events always have a header which stops here (i.e. their
* header is FROZEN).
*/
/*
Initialization to zero of all other Log_event members as they're
not specified. Currently there are no such members; in the future
there will be an event UID (but Format_description and Rotate
don't need this UID, as they are not propagated through
--log-slave-updates (remember the UID is used to not play a query
twice when you have two masters which are slaves of a 3rd master).
Then we are done.
* Initialization to zero of all other Log_event members as they're
* not specified. Currently there are no such members; in the future
* there will be an event UID (but Format_description and Rotate
* don't need this UID, as they are not propagated through
* --log-slave-updates (remember the UID is used to not play a query
* twice when you have two masters which are slaves of a 3rd
* master). Then we are done.
*/
if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) {
@@ -213,25 +193,22 @@ public final class LogHeader
}
/*
CRC verification by SQL and Show-Binlog-Events master side.
The caller has to provide @description_event->checksum_alg to
be the last seen FD's (A) descriptor.
If event is FD the descriptor is in it.
Notice, FD of the binlog can be only in one instance and therefore
Show-Binlog-Events executing master side thread needs just to know
the only FD's (A) value - whereas RL can contain more.
In the RL case, the alg is kept in FD_e (@description_event) which is reset
to the newer read-out event after its execution with possibly new alg descriptor.
Therefore in a typical sequence of RL:
{FD_s^0, FD_m, E_m^1} E_m^1
will be verified with (A) of FD_m.
See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs
lines (log.h).
Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF.
* CRC verification by SQL and Show-Binlog-Events master side. The
* caller has to provide @description_event->checksum_alg to be the last
* seen FD's (A) descriptor. If event is FD the descriptor is in it.
* Notice, FD of the binlog can be only in one instance and therefore
* Show-Binlog-Events executing master side thread needs just to know
* the only FD's (A) value - whereas RL can contain more. In the RL
* case, the alg is kept in FD_e (@description_event) which is reset to
* the newer read-out event after its execution with possibly new alg
* descriptor. Therefore in a typical sequence of RL: {FD_s^0, FD_m,
* E_m^1} E_m^1 will be verified with (A) of FD_m. See legends
* definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs lines
* (log.h). Notice, a pre-checksum FD version forces alg :=
* BINLOG_CHECKSUM_ALG_UNDEF.
*/
checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg
checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch
// checksum alg
processCheckSum(buffer);
/* otherwise, go on with reading the header from buf (nothing now) */
}
@@ -239,8 +216,7 @@ public final class LogHeader
/**
* The different types of log events.
*/
public final int getType()
{
public final int getType() {
return type;
}
@@ -251,8 +227,7 @@ public final class LogHeader
* file. In a relay log, this is the position of the next event in the
* master's binlog.
*/
public final long getLogPos()
{
public final long getLogPos() {
return logPos;
}
@@ -260,24 +235,21 @@ public final class LogHeader
* The total size of this event, in bytes. In other words, this is the sum
* of the sizes of Common-Header, Post-Header, and Body.
*/
public final int getEventLen()
{
public final int getEventLen() {
return eventLen;
}
/**
* The time when the query started, in seconds since 1970.
*/
public final long getWhen()
{
public final long getWhen() {
return when;
}
/**
* Server ID of the server that created the event.
*/
public final long getServerId()
{
public final long getServerId() {
return serverId;
}
@@ -286,24 +258,20 @@ public final class LogHeader
* LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
* LOG_EVENT_SUPPRESS_USE_F for notes.
*/
public final int getFlags()
{
public final int getFlags() {
return flags;
}
public long getCrc() {
return crc;
}
public int getChecksumAlg() {
return checksumAlg;
}
private void processCheckSum(LogBuffer buffer) {
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF &&
checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
}
}

2
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java
View File

@@ -4,7 +4,6 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
*
* @author jianghang 2013-4-8 上午12:36:29
* @version 1.0.3
* @since mysql 5.6
@@ -16,4 +15,3 @@ public class PreviousGtidsLogEvent extends LogEvent{
// do nothing , just for mysql gtid search function
}
}

407
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
View File

@@ -9,19 +9,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* A Query_log_event is created for each query that modifies the database,
* unless the query is logged row-based.
*
* The Post-Header has five components:
*
* unless the query is logged row-based. The Post-Header has five components:
* <table>
* <caption>Post-Header for Query_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>slave_proxy_id</td>
* <td>4 byte unsigned integer</td>
@@ -31,20 +26,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* temporary table local to the client. The slave_proxy_id is used to
* distinguish temporary tables that belong to different clients.</td>
* </tr>
*
* <tr>
* <td>exec_time</td>
* <td>4 byte unsigned integer</td>
* <td>The time from when the query started to when it was logged in the binlog,
* in seconds.</td>
* </tr>
*
* <tr>
* <td>db_len</td>
* <td>1 byte integer</td>
* <td>The length of the name of the currently selected database.</td>
* </tr>
*
* <tr>
* <td>error_code</td>
* <td>2 byte unsigned integer</td>
@@ -52,7 +44,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* fail with the same error code, except for the error codes ER_DB_CREATE_EXISTS
* == 1007 and ER_DB_DROP_EXISTS == 1008.</td>
* </tr>
*
* <tr>
* <td>status_vars_len</td>
* <td>2 byte unsigned integer</td>
@@ -60,18 +51,14 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* query_log_event_status_vars "below".</td>
* </tr>
* </table>
*
* The Body has the following components:
*
* <table>
* <caption>Body for Query_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>query_log_event_status_vars status_vars</td>
* <td>status_vars_len bytes</td>
@@ -81,16 +68,13 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* Table_query_log_event_status_vars "below". MySQL always writes events in the
* order defined below; however, it is capable of reading them in any order.</td>
* </tr>
*
* <tr>
* <td>db</td>
* <td>db_len+1</td>
* <td>The currently selected database, as a null-terminated string.
*
* (The trailing zero is redundant since the length is already known; it is
* db_len from Post-Header.)</td>
* <td>The currently selected database, as a null-terminated string. (The
* trailing zero is redundant since the length is already known; it is db_len
* from Post-Header.)</td>
* </tr>
*
* <tr>
* <td>query</td>
* <td>variable length string without trailing zero, extending to the end of the
@@ -98,20 +82,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>The SQL query.</td>
* </tr>
* </table>
*
* The following table lists the status variables that may appear in the
* status_vars field. Table_query_log_event_status_vars
*
* <table>
* <caption>Status variables for Query_log_event</caption>
*
* <tr>
* <th>Status variable</th>
* <th>1 byte identifier</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>flags2</td>
* <td>Q_FLAGS2_CODE == 0</td>
@@ -121,16 +101,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* identifies those options that need to be written to the binlog (not all do).
* Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL |
* OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
* OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex.
*
* These flags correspond to the SQL variables SQL_AUTO_IS_NULL,
* FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and AUTOCOMMIT, documented in the
* "SET Syntax" section of the MySQL Manual.
*
* This field is always written to the binlog in version >= 5.0, and never
* written in version < 5.0.</td>
* OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. These flags correspond to the
* SQL variables SQL_AUTO_IS_NULL, FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and
* AUTOCOMMIT, documented in the "SET Syntax" section of the MySQL Manual. This
* field is always written to the binlog in version >= 5.0, and never written in
* version < 5.0.</td>
* </tr>
*
* <tr>
* <td>sql_mode</td>
* <td>Q_SQL_MODE_CODE == 1</td>
@@ -177,11 +153,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* All these flags are replicated from the server. However, all flags except
* MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always preserves
* its old value of MODE_NO_DIR_IN_CREATE. For a rationale, see comment in
* Query_log_event::do_apply_event in log_event.cc.
*
* This field is always written to the binlog.</td>
* Query_log_event::do_apply_event in log_event.cc. This field is always written
* to the binlog.</td>
* </tr>
*
* <tr>
* <td>catalog</td>
* <td>Q_CATALOG_NZ_CODE == 6</td>
@@ -189,23 +163,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* characters (at most 255 bytes)</td>
* <td>Stores the client's current catalog. Every database belongs to a catalog,
* the same way that every table belongs to a database. Currently, there is only
* one catalog, "std".
*
* This field is written if the length of the catalog is > 0; otherwise it is
* not written.</td>
* one catalog, "std". This field is written if the length of the catalog is >
* 0; otherwise it is not written.</td>
* </tr>
*
* <tr>
* <td>auto_increment</td>
* <td>Q_AUTO_INCREMENT == 3</td>
* <td>two 2 byte unsigned integers, totally 2+2=4 bytes</td>
*
* <td>The two variables auto_increment_increment and auto_increment_offset, in
* that order. For more information, see "System variables" in the MySQL manual.
*
* This field is written if auto_increment > 1. Otherwise, it is not written.</td>
* </tr>
*
* <tr>
* <td>charset</td>
* <td>Q_CHARSET_CODE == 4</td>
@@ -216,33 +184,22 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* collation_connection identifies the character set and collation that the
* master converts the query to when it receives it; this is useful when
* comparing literal strings. collation_server is the default character set and
* collation used when a new database is created.
*
* See also "Connection Character Sets and Collations" in the MySQL 5.1 manual.
*
* All three variables are codes identifying a (character set, collation) pair.
* To see which codes map to which pairs, run the query "SELECT id,
* character_set_name, collation_name FROM COLLATIONS".
*
* Cf. Q_CHARSET_DATABASE_CODE below.
*
* This field is always written.</td>
* collation used when a new database is created. See also
* "Connection Character Sets and Collations" in the MySQL 5.1 manual. All three
* variables are codes identifying a (character set, collation) pair. To see
* which codes map to which pairs, run the query "SELECT id, character_set_name,
* collation_name FROM COLLATIONS". Cf. Q_CHARSET_DATABASE_CODE below. This
* field is always written.</td>
* </tr>
*
* <tr>
* <td>time_zone</td>
* <td>Q_TIME_ZONE_CODE == 5</td>
* <td>Variable-length string: the length in bytes (1 byte) followed by the
* characters (at most 255 bytes).
* <td>The time_zone of the master.
*
* See also "System Variables" and "MySQL Server Time Zone Support" in the MySQL
* manual.
*
* This field is written if the length of the time zone string is > 0;
* otherwise, it is not written.</td>
* <td>The time_zone of the master. See also "System Variables" and
* "MySQL Server Time Zone Support" in the MySQL manual. This field is written
* if the length of the time zone string is > 0; otherwise, it is not written.</td>
* </tr>
*
* <tr>
* <td>lc_time_names_number</td>
* <td>Q_LC_TIME_NAMES_CODE == 7</td>
@@ -251,84 +208,68 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* to languages is defined in sql_locale.cc. This field is written if it is not
* 0, i.e., if the locale is not en_US.</td>
* </tr>
*
* <tr>
* <td>charset_database_number</td>
* <td>Q_CHARSET_DATABASE_CODE == 8</td>
* <td>2 byte integer</td>
*
* <td>The value of the collation_database system variable (in the source code
* stored in thd->variables.collation_database), which holds the code for a
* (character set, collation) pair as described above (see Q_CHARSET_CODE).
*
* collation_database was used in old versions (???WHEN). Its value was loaded
* when issuing a "use db" query and could be changed by issuing a
* "SET collation_database=xxx" query. It used to affect the "LOAD DATA INFILE"
* and "CREATE TABLE" commands.
*
* In newer versions, "CREATE TABLE" has been changed to take the character set
* from the database of the created table, rather than the character set of the
* current database. This makes a difference when creating a table in another
* database than the current one. "LOAD DATA INFILE" has not yet changed to do
* this, but there are plans to eventually do it, and to make collation_database
* read-only.
*
* This field is written if it is not 0.</td>
* and "CREATE TABLE" commands. In newer versions, "CREATE TABLE" has been
* changed to take the character set from the database of the created table,
* rather than the character set of the current database. This makes a
* difference when creating a table in another database than the current one.
* "LOAD DATA INFILE" has not yet changed to do this, but there are plans to
* eventually do it, and to make collation_database read-only. This field is
* written if it is not 0.</td>
* </tr>
* <tr>
* <td>table_map_for_update</td>
* <td>Q_TABLE_MAP_FOR_UPDATE_CODE == 9</td>
* <td>8 byte integer</td>
*
* <td>The value of the table map that is to be updated by the multi-table
* update query statement. Every bit of this variable represents a table, and is
* set to 1 if the corresponding table is to be updated by this statement.
*
* The value of this variable is set when executing a multi-table update
* statement and used by slave to apply filter rules without opening all the
* tables on slave. This is required because some tables may not exist on slave
* because of the filter rules.</td>
* set to 1 if the corresponding table is to be updated by this statement. The
* value of this variable is set when executing a multi-table update statement
* and used by slave to apply filter rules without opening all the tables on
* slave. This is required because some tables may not exist on slave because of
* the filter rules.</td>
* </tr>
* </table>
*
* Query_log_event_notes_on_previous_versions Notes on Previous Versions
*
* Status vars were introduced in version 5.0. To read earlier versions
* correctly, check the length of the Post-Header.
*
* The status variable Q_CATALOG_CODE == 2 existed in MySQL 5.0.x, where
* 0<=x<=3. It was identical to Q_CATALOG_CODE, except that the string had a
* trailing '\0'. The '\0' was removed in 5.0.4 since it was redundant (the
* string length is stored before the string). The Q_CATALOG_CODE will never be
* written by a new master, but can still be understood by a new slave.
*
* See Q_CHARSET_DATABASE_CODE in the table above.
*
* Query_log_event_notes_on_previous_versions Notes on Previous Versions Status
* vars were introduced in version 5.0. To read earlier versions correctly,
* check the length of the Post-Header. The status variable Q_CATALOG_CODE == 2
* existed in MySQL 5.0.x, where 0<=x<=3. It was identical to Q_CATALOG_CODE,
* except that the string had a trailing '\0'. The '\0' was removed in 5.0.4
* since it was redundant (the string length is stored before the string). The
* Q_CATALOG_CODE will never be written by a new master, but can still be
* understood by a new slave. See Q_CHARSET_DATABASE_CODE in the table above.
* When adding new status vars, please don't forget to update the
* MAX_SIZE_LOG_EVENT_STATUS, and update function code_name
*
* @see mysql-5.1.6/sql/logevent.cc - Query_log_event
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class QueryLogEvent extends LogEvent
{
public class QueryLogEvent extends LogEvent {
/**
The maximum number of updated databases that a status of
Query-log-event can carry. It can redefined within a range
[1.. OVER_MAX_DBS_IN_EVENT_MTS].
* The maximum number of updated databases that a status of Query-log-event
* can carry. It can redefined within a range [1..
* OVER_MAX_DBS_IN_EVENT_MTS].
*/
public static final int MAX_DBS_IN_EVENT_MTS = 16;
/**
When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS
the value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the
mts_accessed_dbs status.
* When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS the
* value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the mts_accessed_dbs
* status.
*/
public static final int OVER_MAX_DBS_IN_EVENT_MTS = 254;
public static final int SYSTEM_CHARSET_MBMAXLEN = 3;
public static final int NAME_CHAR_LEN = 64;
/* Field/table name length */
@@ -340,23 +281,68 @@ public class QueryLogEvent extends LogEvent
* log_event status vars estimation:
*/
public static final int MAX_SIZE_LOG_EVENT_STATUS = (1 + 4 /* type, flags2 */
+ 1 + 8 /* type, sql_mode */
+ 1 + 1 + 255 /* type, length, catalog */
+ 1 + 4 /* type, auto_increment */
+ 1 + 6 /* type, charset */
+ 1 + 1 + 255 /* type, length, time_zone */
+ 1 + 2 /* type, lc_time_names_number */
+ 1 + 2 /* type, charset_database_number */
+ 1 + 8 /* type, table_map_for_update */
+ 1 + 4 /* type, master_data_written */
/* type, db_1, db_2, ... */
+ 1 + 8 /*
* type,
* sql_mode
*/
+ 1 + 1 + 255 /*
* type,
* length
* ,
* catalog
*/
+ 1 + 4 /*
* type,
* auto_increment
*/
+ 1 + 6 /*
* type,
* charset
*/
+ 1 + 1 + 255 /*
* type,
* length
* ,
* time_zone
*/
+ 1 + 2 /*
* type,
* lc_time_names_number
*/
+ 1 + 2 /*
* type,
* charset_database_number
*/
+ 1 + 8 /*
* type,
* table_map_for_update
*/
+ 1 + 4 /*
* type,
* master_data_written
*/
/*
* type, db_1, db_2,
* ...
*/
/* type, microseconds */
/* MariaDb type, sec_part of NOW() */
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3
+ 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */);
/*
* MariaDb type,
* sec_part of NOW()
*/
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 16 + 1 + 60/*
* type
* ,
* user_len
* ,
* user
* ,
* host_len
* ,
* host
*/);
/**
* Fixed data part:
*
* <ul>
* <li>4 bytes. The ID of the thread that issued this statement. Needed for
* temporary tables. This is also useful for a DBA for knowing who did what
@@ -385,7 +371,6 @@ public class QueryLogEvent extends LogEvent
* <li>2 bytes (not present in v1, v3). The length of the status variable
* block.</li>
* </ul>
*
* Variable part:
* <ul>
* <li>Zero or more status variables (not present in v1, v3). Each status
@@ -397,7 +382,6 @@ public class QueryLogEvent extends LogEvent
* the variable part (the sizes are given in the fixed data part), so by
* subtraction it can know the size of the statement.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private String user;
@@ -409,7 +393,8 @@ public class QueryLogEvent extends LogEvent
protected final String dbname;
/** The number of seconds the query took to run on the master. */
// The time in seconds that the statement took to execute. Only useful for inspection by the DBA
// The time in seconds that the statement took to execute. Only useful for
// inspection by the DBA
private final long execTime;
private final int errorCode;
private final long sessionId; /* thread_id */
@@ -434,9 +419,8 @@ public class QueryLogEvent extends LogEvent
private String timezone;
public QueryLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent) throws IOException
{
public QueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
@@ -447,8 +431,7 @@ public class QueryLogEvent extends LogEvent
* format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same
* for 3.23, 4.0 & 5.0.
*/
if (buffer.limit() < (commonHeaderLen + postHeaderLen))
{
if (buffer.limit() < (commonHeaderLen + postHeaderLen)) {
throw new IOException("Query event length is too short.");
}
int dataLen = buffer.limit() - (commonHeaderLen + postHeaderLen);
@@ -462,24 +445,21 @@ public class QueryLogEvent extends LogEvent
errorCode = buffer.getUint16(); // Q_ERR_CODE_OFFSET
/*
* 5.0 format starts here. Depending on the format, we may or not
* have affected/warnings etc The remaining post-header to be parsed
* has length:
* 5.0 format starts here. Depending on the format, we may or not have
* affected/warnings etc The remaining post-header to be parsed has
* length:
*/
int statusVarsLen = 0;
if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN)
{
if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN) {
statusVarsLen = buffer.getUint16(); // Q_STATUS_VARS_LEN_OFFSET
/*
Check if status variable length is corrupt and will lead to very
wrong data. We could be even more strict and require data_len to
be even bigger, but this will suffice to catch most corruption
errors that can lead to a crash.
* Check if status variable length is corrupt and will lead to very
* wrong data. We could be even more strict and require data_len to
* be even bigger, but this will suffice to catch most corruption
* errors that can lead to a crash.
*/
if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS))
{
throw new IOException("status_vars_len (" + statusVarsLen
+ ") > data_len (" + dataLen + ")");
if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS)) {
throw new IOException("status_vars_len (" + statusVarsLen + ") > data_len (" + dataLen + ")");
}
dataLen -= statusVarsLen;
}
@@ -501,27 +481,19 @@ public class QueryLogEvent extends LogEvent
/* A 2nd variable part; this is common to all versions */
final int queryLen = dataLen - dbLen - 1;
dbname = buffer.getFixString(dbLen + 1);
if (clientCharset >= 0)
{
if (clientCharset >= 0) {
charsetName = CharsetConversion.getJavaCharset(clientCharset);
if ((charsetName != null) && (Charset.isSupported(charsetName)))
{
if ((charsetName != null) && (Charset.isSupported(charsetName))) {
query = buffer.getFixString(queryLen, charsetName);
}
else
{
logger.warn("unsupported character set in query log: "
+ "\n ID = " + clientCharset + ", Charset = "
+ CharsetConversion.getCharset(clientCharset)
+ ", Collation = "
} else {
logger.warn("unsupported character set in query log: " + "\n ID = " + clientCharset + ", Charset = "
+ CharsetConversion.getCharset(clientCharset) + ", Collation = "
+ CharsetConversion.getCollation(clientCharset));
query = buffer.getFixString(queryLen);
}
}
else
{
} else {
query = buffer.getFixString(queryLen);
}
}
@@ -569,9 +541,9 @@ public class QueryLogEvent extends LogEvent
public static final int Q_INVOKER = 11;
/**
Q_UPDATED_DB_NAMES status variable collects of the updated databases
total number and their names to be propagated to the slave in order
to facilitate the parallel applying of the Query events.
* Q_UPDATED_DB_NAMES status variable collects of the updated databases
* total number and their names to be propagated to the slave in order to
* facilitate the parallel applying of the Query events.
*/
public static final int Q_UPDATED_DB_NAMES = 12;
@@ -582,21 +554,17 @@ public class QueryLogEvent extends LogEvent
*/
public static final int Q_HRNOW = 128;
private final void unpackVariables(LogBuffer buffer, final int end)
throws IOException
{
private final void unpackVariables(LogBuffer buffer, final int end) throws IOException {
int code = -1;
try
{
while (buffer.position() < end)
{
switch (code = buffer.getUint8())
{
try {
while (buffer.position() < end) {
switch (code = buffer.getUint8()) {
case Q_FLAGS2_CODE:
flags2 = buffer.getUint32();
break;
case Q_SQL_MODE_CODE:
sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode is ulonglong
sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode
// is ulonglong
break;
case Q_CATALOG_NZ_CODE:
catalog = buffer.getString();
@@ -634,7 +602,8 @@ public class QueryLogEvent extends LogEvent
buffer.forward(8);
break;
case Q_MASTER_DATA_WRITTEN_CODE:
// data_written = master_data_written = buffer.getUint32();
// data_written = master_data_written =
// buffer.getUint32();
buffer.forward(4);
break;
case Q_INVOKER:
@@ -648,9 +617,10 @@ public class QueryLogEvent extends LogEvent
case Q_UPDATED_DB_NAMES:
int mtsAccessedDbs = buffer.getUint8();
/**
Notice, the following check is positive also in case of
the master's MAX_DBS_IN_EVENT_MTS > the slave's one and the event
contains e.g the master's MAX_DBS_IN_EVENT_MTS db:s.
* Notice, the following check is positive also in case
* of the master's MAX_DBS_IN_EVENT_MTS > the slave's
* one and the event contains e.g the master's
* MAX_DBS_IN_EVENT_MTS db:s.
*/
if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) {
mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS;
@@ -667,25 +637,22 @@ public class QueryLogEvent extends LogEvent
buffer.forward(3);
break;
default:
/* That's why you must write status vars in growing order of code */
if (logger.isDebugEnabled())
logger.debug("Query_log_event has unknown status vars (first has code: "
/*
* That's why you must write status vars in growing
* order of code
*/
if (logger.isDebugEnabled()) logger.debug("Query_log_event has unknown status vars (first has code: "
+ code + "), skipping the rest of them");
break; // Break loop
}
}
}
catch (RuntimeException e)
{
throw new IOException("Read " + findCodeName(code) + " error: "
+ e.getMessage(), e);
} catch (RuntimeException e) {
throw new IOException("Read " + findCodeName(code) + " error: " + e.getMessage(), e);
}
}
private static final String findCodeName(final int code)
{
switch (code)
{
private static final String findCodeName(final int code) {
switch (code) {
case Q_FLAGS2_CODE:
return "Q_FLAGS2_CODE";
case Q_SQL_MODE_CODE:
@@ -716,66 +683,54 @@ public class QueryLogEvent extends LogEvent
return "CODE#" + code;
}
public final String getUser()
{
public final String getUser() {
return user;
}
public final String getHost()
{
public final String getHost() {
return host;
}
public final String getQuery()
{
public final String getQuery() {
return query;
}
public final String getCatalog()
{
public final String getCatalog() {
return catalog;
}
public final String getDbName()
{
public final String getDbName() {
return dbname;
}
/**
* The number of seconds the query took to run on the master.
*/
public final long getExecTime()
{
public final long getExecTime() {
return execTime;
}
public final int getErrorCode()
{
public final int getErrorCode() {
return errorCode;
}
public final long getSessionId()
{
public final long getSessionId() {
return sessionId;
}
public final long getAutoIncrementIncrement()
{
public final long getAutoIncrementIncrement() {
return autoIncrementIncrement;
}
public final long getAutoIncrementOffset()
{
public final long getAutoIncrementOffset() {
return autoIncrementOffset;
}
public final String getCharsetName()
{
public final String getCharsetName() {
return charsetName;
}
public final String getTimezone()
{
public final String getTimezone() {
return timezone;
}
@@ -784,8 +739,7 @@ public class QueryLogEvent extends LogEvent
*
* @return Returns the charsetID.
*/
public final int getClientCharset()
{
public final int getClientCharset() {
return clientCharset;
}
@@ -794,8 +748,7 @@ public class QueryLogEvent extends LogEvent
*
* @return Returns the clientCollationId.
*/
public final int getClientCollation()
{
public final int getClientCollation() {
return clientCollation;
}
@@ -804,19 +757,16 @@ public class QueryLogEvent extends LogEvent
*
* @return Returns the serverCollationId.
*/
public final int getServerCollation()
{
public final int getServerCollation() {
return serverCollation;
}
/**
* Returns the sql_mode value.
*
* <p>
* The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
* and see mysql_priv.h for a list of the possible flags. Currently
* (2007-10-04), the following flags are available:
*
* <ul>
* <li>MODE_REAL_AS_FLOAT==0x1</li>
* <li>MODE_PIPES_AS_CONCAT==0x2</li>
@@ -851,14 +801,12 @@ public class QueryLogEvent extends LogEvent
* <li>MODE_NO_ENGINE_SUBSTITUTION=0x40000000</li>
* <li>MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000</li>
* </ul>
*
* All these flags are replicated from the server. However, all flags except
* MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always
* preserves its old value of MODE_NO_DIR_IN_CREATE. This field is always
* written to the binlog.
*/
public final long getSqlMode()
{
public final long getSqlMode() {
return sql_mode;
}
@@ -880,40 +828,35 @@ public class QueryLogEvent extends LogEvent
* Manual. This field is always written to the binlog in version >= 5.0, and
* never written in version < 5.0.
*/
public final long getFlags2()
{
public final long getFlags2() {
return flags2;
}
/**
* Returns the OPTION_AUTO_IS_NULL flag.
*/
public final boolean isAutoIsNull()
{
public final boolean isAutoIsNull() {
return ((flags2 & OPTION_AUTO_IS_NULL) == OPTION_AUTO_IS_NULL);
}
/**
* Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
*/
public final boolean isForeignKeyChecks()
{
public final boolean isForeignKeyChecks() {
return ((flags2 & OPTION_NO_FOREIGN_KEY_CHECKS) != OPTION_NO_FOREIGN_KEY_CHECKS);
}
/**
* Returns the OPTION_NOT_AUTOCOMMIT flag.
*/
public final boolean isAutocommit()
{
public final boolean isAutocommit() {
return ((flags2 & OPTION_NOT_AUTOCOMMIT) != OPTION_NOT_AUTOCOMMIT);
}
/**
* Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
*/
public final boolean isUniqueChecks()
{
public final boolean isUniqueChecks() {
return ((flags2 & OPTION_RELAXED_UNIQUE_CHECKS) != OPTION_RELAXED_UNIQUE_CHECKS);
}

30
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java
View File

@@ -7,30 +7,21 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1
* does not need it (it's repeatable again) so this event needn't be written in
* 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it
* does not cause bugs).
*
* The state of the random number generation consists of 128 bits, which are
* stored internally as two 64-bit numbers.
*
* Binary Format
*
* does not cause bugs). The state of the random number generation consists of
* 128 bits, which are stored internally as two 64-bit numbers. Binary Format
* The Post-Header for this event type is empty. The Body has two components:
*
* <table>
* <caption>Body for Rand_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>seed1</td>
* <td>8 byte unsigned integer</td>
* <td>64 bit random seed1.</td>
* </tr>
*
* <tr>
* <td>seed2</td>
* <td>8 byte unsigned integer</td>
@@ -41,19 +32,16 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class RandLogEvent extends LogEvent
{
public final class RandLogEvent extends LogEvent {
/**
* Fixed data part: Empty
*
* <p>
* Variable data part:
*
* <ul>
* <li>8 bytes. The value for the first seed.</li>
* <li>8 bytes. The value for the second seed.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final long seed1;
@@ -63,21 +51,17 @@ public final class RandLogEvent extends LogEvent
public static final int RAND_SEED1_OFFSET = 0;
public static final int RAND_SEED2_OFFSET = 8;
public RandLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public RandLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
/* The Post-Header is empty. The Variable Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[RAND_EVENT - 1]
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
+ RAND_SEED1_OFFSET);
seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET);
seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET);
}
public final String getQuery()
{
public final String getQuery() {
return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2;
}
}

47
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java
View File

@@ -4,57 +4,44 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* This will be deprecated when we move to using sequence ids.
*
* Binary Format
*
* The Post-Header has one component:
*
* This will be deprecated when we move to using sequence ids. Binary Format The
* Post-Header has one component:
* <table>
* <caption>Post-Header for Rotate_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>position</td>
* <td>8 byte integer</td>
* <td>The position within the binlog to rotate to.</td>
* </tr>
*
* </table>
*
* The Body has one component:
*
* <table>
* <caption>Body for Rotate_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>new_log</td>
* <td>variable length string without trailing zero, extending to the end of the
* event (determined by the length field of the Common-Header)</td>
* <td>Name of the binlog to rotate to.</td>
* </tr>
*
* </table>
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class RotateLogEvent extends LogEvent
{
public final class RotateLogEvent extends LogEvent {
/**
* Fixed data part:
*
* <ul>
* <li>8 bytes. The position of the first event in the next log file. Always
* contains the number 4 (meaning the next event starts at position 4 in the
@@ -62,14 +49,11 @@ public final class RotateLogEvent extends LogEvent
* is assumed to be 4.</li>
* </ul>
* <p>
*
* Variable data part:
*
* <ul>
* <li>The name of the next binary log. The filename is not null-terminated.
* Its length is the event size minus the size of the fixed parts.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final String filename;
@@ -91,21 +75,20 @@ public final class RotateLogEvent extends LogEvent
*
* @throws MySQLExtractException
*/
public RotateLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public RotateLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int headerSize = descriptionEvent.commonHeaderLen;
final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1];
buffer.position(headerSize + R_POS_OFFSET);
position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET)
position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf
// +
// R_POS_OFFSET)
final int filenameOffset = headerSize + postHeaderLen;
int filenameLen = buffer.limit() - filenameOffset;
if (filenameLen > FN_REFLEN - 1)
filenameLen = FN_REFLEN - 1;
if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1;
buffer.position(filenameOffset);
filename = buffer.getFixString(filenameLen);
}
@@ -114,8 +97,7 @@ public final class RotateLogEvent extends LogEvent
* Creates a new <code>Rotate_log_event</code> without log information. This
* is used to generate missing log rotation events.
*/
public RotateLogEvent(String filename)
{
public RotateLogEvent(String filename){
super(ROTATE_HEADER);
this.filename = filename;
@@ -125,21 +107,18 @@ public final class RotateLogEvent extends LogEvent
/**
* Creates a new <code>Rotate_log_event</code> without log information.
*/
public RotateLogEvent(String filename, final long position)
{
public RotateLogEvent(String filename, final long position){
super(ROTATE_HEADER);
this.filename = filename;
this.position = position;
}
public final String getFilename()
{
public final String getFilename() {
return filename;
}
public final long getPosition()
{
public final long getPosition() {
return position;
}
}

90
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java
View File

@@ -12,41 +12,33 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public abstract class RowsLogEvent extends LogEvent
{
public abstract class RowsLogEvent extends LogEvent {
/**
* Fixed data part:
*
* <ul>
* <li>6 bytes. The table ID.</li>
* <li>2 bytes. Reserved for future use.</li>
* </ul>
*
* <p>
* Variable data part:
*
* <ul>
* <li>Packed integer. The number of columns in the table.</li>
*
* <li>Variable-sized. Bit-field indicating whether each column is used, one
* bit per column. For this field, the amount of storage required for N
* columns is INT((N+7)/8) bytes.</li>
*
* <li>Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating
* whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one
* bit per column. For this field, the amount of storage required for N
* columns is INT((N+7)/8) bytes.</li>
*
* <li>Variable-sized. A sequence of zero or more rows. The end is
* determined by the size of the event. Each row has the following format:
*
* <ul>
* <li>Variable-sized. Bit-field indicating whether each field in the row is
* NULL. Only columns that are "used" according to the second field in the
* variable data part are listed here. If the second field in the variable
* data part has N one-bits, the amount of storage required for this field
* is INT((N+7)/8) bytes.</li>
*
* <li>Variable-sized. The row-image, containing values of all table fields.
* This only lists table fields that are used (according to the second field
* of the variable data part) and non-NULL (according to the previous
@@ -54,16 +46,18 @@ public abstract class RowsLogEvent extends LogEvent
* number of zero bits in the previous field (not counting padding bits in
* the last byte). The format of each value is described in the
* log_event_print_value() function in log_event.cc.</li>
*
* <li>(for UPDATE_ROWS_EVENT only) the previous two fields are repeated,
* representing a second table row.</li>
* </ul>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final long tableId; /* Table ID */
private TableMapLogEvent table; /* The table the rows belong to */
private TableMapLogEvent table; /*
* The table
* the rows
* belong to
*/
/** Bitmap denoting columns available */
protected final int columnLen;
@@ -78,15 +72,17 @@ public abstract class RowsLogEvent extends LogEvent
protected final BitSet changeColumns;
/** XXX: Don't handle buffer in another thread. */
private final LogBuffer rowsBuf; /* The rows in packed format */
private final LogBuffer rowsBuf; /*
* The rows in
* packed
* format
*/
/**
* enum enum_flag
*
* These definitions allow you to combine the flags into an appropriate flag
* set using the normal bitwise operators. The implicit conversion from an
* enum-constant to an integer is accepted by the compiler, which is then
* used to set the real set of flags.
* enum enum_flag These definitions allow you to combine the flags into an
* appropriate flag set using the normal bitwise operators. The implicit
* conversion from an enum-constant to an integer is accepted by the
* compiler, which is then used to set the real set of flags.
*/
private final int flags;
@@ -112,29 +108,25 @@ public abstract class RowsLogEvent extends LogEvent
public static final int RW_V_TAG_LEN = 1;
public static final int RW_V_EXTRAINFO_TAG = 0;
public RowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public RowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
int headerLen = 0;
buffer.position(commonHeaderLen + RW_MAPID_OFFSET);
if (postHeaderLen == 6)
{
/* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
if (postHeaderLen == 6) {
/*
* Master is of an intermediate source tree before 5.1.4. Id is 4
* bytes
*/
tableId = buffer.getUint32();
}
else
{
} else {
tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET
}
flags = buffer.getUint16();
if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2)
{
if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2) {
headerLen = buffer.getUint16();
headerLen -= 2;
int start = buffer.position();
@@ -148,7 +140,8 @@ public abstract class RowsLogEvent extends LogEvent
int val = checkLen - EXTRA_ROW_INFO_HDR_BYTES;
assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_FORMAT_OFFSET
for (int j = 0; j < val; j++) {
assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i
assert (buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES
// + i
}
break;
default:
@@ -162,12 +155,9 @@ public abstract class RowsLogEvent extends LogEvent
columnLen = (int) buffer.getPackedLong();
columns = buffer.getBitmap(columnLen);
if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT)
{
if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) {
changeColumns = buffer.getBitmap(columnLen);
}
else
{
} else {
changeColumns = columns;
}
@@ -176,46 +166,38 @@ public abstract class RowsLogEvent extends LogEvent
rowsBuf = buffer.duplicate(dataSize);
}
public final void fillTable(LogContext context)
{
public final void fillTable(LogContext context) {
table = context.getTable(tableId);
// end of statement check:
if ((flags & RowsLogEvent.STMT_END_F) != 0)
{
if ((flags & RowsLogEvent.STMT_END_F) != 0) {
// Now is safe to clear ignored map (clear_tables will also
// delete original table map events stored in the map).
context.clearAllTables();
}
}
public final long getTableId()
{
public final long getTableId() {
return tableId;
}
public final TableMapLogEvent getTable()
{
public final TableMapLogEvent getTable() {
return table;
}
public final BitSet getColumns()
{
public final BitSet getColumns() {
return columns;
}
public final BitSet getChangeColumns()
{
public final BitSet getChangeColumns() {
return changeColumns;
}
public final RowsLogBuffer getRowsBuf(String charsetName)
{
public final RowsLogBuffer getRowsBuf(String charsetName) {
return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName);
}
public final int getFlags(final int flags)
{
public final int getFlags(final int flags) {
return this.flags & flags;
}
}

22
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java
View File

@@ -5,21 +5,18 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and
* 4.x).
*
* Format_description_log_event derives from Start_log_event_v3; it is the
* 4.x). Format_description_log_event derives from Start_log_event_v3; it is the
* Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that
* describes the other events' Common-Header/Post-Header lengths. This event is
* sent by MySQL 5.0 whenever it starts sending a new binlog if the requested
* position is >4 (otherwise if ==4 the event will be sent naturally).
*
* @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public class StartLogEventV3 extends LogEvent
{
public class StartLogEventV3 extends LogEvent {
/**
* We could have used SERVER_VERSION_LENGTH, but this introduces an obscure
* dependency - if somebody decided to change SERVER_VERSION_LENGTH this
@@ -34,9 +31,7 @@ public class StartLogEventV3 extends LogEvent
protected int binlogVersion;
protected String serverVersion;
public StartLogEventV3(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public StartLogEventV3(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
buffer.position(descriptionEvent.commonHeaderLen);
@@ -44,18 +39,15 @@ public class StartLogEventV3 extends LogEvent
serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
}
public StartLogEventV3()
{
public StartLogEventV3(){
super(new LogHeader(START_EVENT_V3));
}
public final String getServerVersion()
{
public final String getServerVersion() {
return serverVersion;
}
public final int getBinlogVersion()
{
public final int getBinlogVersion() {
return binlogVersion;
}
}

14
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java
View File

@@ -4,19 +4,15 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* Stop_log_event.
*
* The Post-Header and Body for this event type are empty; it only has the
* Common-Header.
* Stop_log_event. The Post-Header and Body for this event type are empty; it
* only has the Common-Header.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class StopLogEvent extends LogEvent
{
public StopLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent description_event)
{
public final class StopLogEvent extends LogEvent {
public StopLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent description_event){
super(header);
}
}

121
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
View File

@@ -8,44 +8,34 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* In row-based mode, every row operation event is preceded by a
* Table_map_log_event which maps a table definition to a number. The table
* definition consists of database name, table name, and column definitions.
*
* The Post-Header has the following components:
*
* definition consists of database name, table name, and column definitions. The
* Post-Header has the following components:
* <table>
* <caption>Post-Header for Table_map_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>table_id</td>
* <td>6 bytes unsigned integer</td>
* <td>The number that identifies the table.</td>
* </tr>
*
* <tr>
* <td>flags</td>
* <td>2 byte bitfield</td>
* <td>Reserved for future use; currently always 0.</td>
* </tr>
*
* </table>
*
* The Body has the following components:
*
* <table>
* <caption>Body for Table_map_log_event</caption>
*
* <tr>
* <th>Name</th>
* <th>Format</th>
* <th>Description</th>
* </tr>
*
* <tr>
* <td>database_name</td>
* <td>one byte string length, followed by null-terminated string</td>
@@ -55,20 +45,17 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* by a terminating 0 byte. (Note the redundancy in the representation of the
* length.)</td>
* </tr>
*
* <tr>
* <td>table_name</td>
* <td>one byte string length, followed by null-terminated string</td>
* <td>The name of the table, encoded the same way as the database name above.</td>
* </tr>
*
* <tr>
* <td>column_count</td>
* <td>packed_integer "Packed Integer"</td>
* <td>The number of columns in the table, represented as a packed
* variable-length integer.</td>
* </tr>
*
* <tr>
* <td>column_type</td>
* <td>List of column_count 1 byte enumeration values</td>
@@ -78,13 +65,11 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* listed in the table Table_table_map_log_event_column_types "below" (along
* with description of the associated metadata field).</td>
* </tr>
*
* <tr>
* <td>metadata_length</td>
* <td>packed_integer "Packed Integer"</td>
* <td>The length of the following metadata block</td>
* </tr>
*
* <tr>
* <td>metadata</td>
* <td>list of metadata for each column</td>
@@ -93,7 +78,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* metadata for each column are listed in the table
* Table_table_map_log_event_column_types "below".</td>
* </tr>
*
* <tr>
* <td>null_bits</td>
* <td>column_count bits, rounded up to nearest byte</td>
@@ -104,13 +88,10 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* first byte, the ninth is in the least significant bit of the second byte, and
* so on.</td>
* </tr>
*
* </table>
*
* The table below lists all column types, along with the numerical identifier
* for it and the size and interpretation of meta-data used to describe the
* type.
*
* <table>
* <caption>Table_map_log_event column types: numerical identifier and
* metadata</caption>
@@ -120,35 +101,30 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <th>Size of metadata in bytes</th>
* <th>Description of metadata</th>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DECIMAL</td>
* <td>0</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TINY</td>
* <td>1</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_SHORT</td>
* <td>2</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_LONG</td>
* <td>3</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_FLOAT</td>
* <td>4</td>
@@ -156,7 +132,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>1 byte unsigned integer, representing the "pack_length", which is equal
* to sizeof(float) on the server from which the event originates.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DOUBLE</td>
* <td>5</td>
@@ -164,63 +139,54 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>1 byte unsigned integer, representing the "pack_length", which is equal
* to sizeof(double) on the server from which the event originates.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_NULL</td>
* <td>6</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TIMESTAMP</td>
* <td>7</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_LONGLONG</td>
* <td>8</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_INT24</td>
* <td>9</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DATE</td>
* <td>10</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TIME</td>
* <td>11</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_DATETIME</td>
* <td>12</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_YEAR</td>
* <td>13</td>
* <td>0</td>
* <td>No column metadata.</td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_NEWDATE</i></td>
* <td><i>14</i></td>
@@ -228,14 +194,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_VARCHAR</td>
* <td>15</td>
* <td>2 bytes</td>
* <td>2 byte unsigned integer representing the maximum length of the string.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_BIT</td>
* <td>16</td>
@@ -245,7 +209,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* occupied by the bitfield. The number of bytes is either int((length+7)/8) or
* int(length/8).</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_NEWDECIMAL</td>
* <td>246</td>
@@ -253,7 +216,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>A 1 byte unsigned int representing the precision, followed by a 1 byte
* unsigned int representing the number of decimals.</td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_ENUM</i></td>
* <td><i>247</i></td>
@@ -261,7 +223,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_SET</i></td>
* <td><i>248</i></td>
@@ -269,7 +230,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_TINY_BLOB</td>
* <td>249</td>
@@ -277,7 +237,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_MEDIUM_BLOB</i></td>
* <td><i>250</i></td>
@@ -285,7 +244,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td><i>MYSQL_TYPE_LONG_BLOB</i></td>
* <td><i>251</i></td>
@@ -293,7 +251,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td><i>This enumeration value is only used internally and cannot exist in a
* binlog.</i></td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_BLOB</td>
* <td>252</td>
@@ -301,7 +258,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>The pack length, i.e., the number of bytes needed to represent the length
* of the blob: 1, 2, 3, or 4.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_VAR_STRING</td>
* <td>253</td>
@@ -312,7 +268,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* unsigned integer representing the field size, i.e., the number of bytes
* needed to store the length of the string.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_STRING</td>
* <td>254</td>
@@ -321,7 +276,6 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* byte is the field size, i.e., the number of bytes in the representation of
* size of the string: 3 or 4.</td>
* </tr>
*
* <tr>
* <td>MYSQL_TYPE_GEOMETRY</td>
* <td>255</td>
@@ -329,21 +283,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* <td>The pack length, i.e., the number of bytes needed to represent the length
* of the geometry: 1, 2, 3, or 4.</td>
* </tr>
*
* </table>
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class TableMapLogEvent extends LogEvent
{
public final class TableMapLogEvent extends LogEvent {
/**
* Fixed data part:
* <ul>
* <li>6 bytes. The table ID.</li>
* <li>2 bytes. Reserved for future use.</li>
* </ul>
*
* <p>
* Variable data part:
* <ul>
@@ -360,7 +312,6 @@ public final class TableMapLogEvent extends LogEvent
* one bit per column. For this field, the amount of storage required for N
* columns is INT((N+7)/8) bytes.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
protected final String dbname;
@@ -369,14 +320,15 @@ public final class TableMapLogEvent extends LogEvent
/**
* Holding mysql column information.
*/
public static final class ColumnInfo
{
public static final class ColumnInfo {
public int type;
public int meta;
}
protected final int columnCnt;
protected final ColumnInfo[] columnInfo; // buffer for field metadata
protected final ColumnInfo[] columnInfo; // buffer for field
// metadata
protected final long tableId;
protected BitSet nullBits;
@@ -388,22 +340,20 @@ public final class TableMapLogEvent extends LogEvent
/**
* Constructor used by slave to read the event from the binary log.
*/
public TableMapLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public TableMapLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
/* Read the post-header */
buffer.position(commonHeaderLen + TM_MAPID_OFFSET);
if (postHeaderLen == 6)
{
/* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
if (postHeaderLen == 6) {
/*
* Master is of an intermediate source tree before 5.1.4. Id is 4
* bytes
*/
tableId = buffer.getUint32();
}
else
{
} else {
// DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN);
tableId = buffer.getUlong48();
}
@@ -419,15 +369,13 @@ public final class TableMapLogEvent extends LogEvent
// Read column information from buffer
columnCnt = (int) buffer.getPackedLong();
columnInfo = new ColumnInfo[columnCnt];
for (int i = 0; i < columnCnt; i++)
{
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = new ColumnInfo();
info.type = buffer.getUint8();
columnInfo[i] = info;
}
if (buffer.position() < buffer.limit())
{
if (buffer.position() < buffer.limit()) {
final int fieldSize = (int) buffer.getPackedLong();
decodeFields(buffer, fieldSize);
nullBits = buffer.getBitmap(columnCnt);
@@ -439,17 +387,14 @@ public final class TableMapLogEvent extends LogEvent
*
* @see mysql-5.1.60/sql/rpl_utility.h
*/
private final void decodeFields(LogBuffer buffer, final int len)
{
private final void decodeFields(LogBuffer buffer, final int len) {
final int limit = buffer.limit();
buffer.limit(len + buffer.position());
for (int i = 0; i < columnCnt; i++)
{
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = columnInfo[i];
switch (info.type)
{
switch (info.type) {
case MYSQL_TYPE_TINY_BLOB:
case MYSQL_TYPE_BLOB:
case MYSQL_TYPE_MEDIUM_BLOB:
@@ -458,7 +403,7 @@ public final class TableMapLogEvent extends LogEvent
case MYSQL_TYPE_FLOAT:
case MYSQL_TYPE_GEOMETRY:
/*
These types store a single byte.
* These types store a single byte.
*/
info.meta = buffer.getUint8();
break;
@@ -472,8 +417,7 @@ public final class TableMapLogEvent extends LogEvent
logger.warn("This enumeration value is only used internally "
+ "and cannot exist in a binlog: type=" + info.type);
break;
case MYSQL_TYPE_STRING:
{
case MYSQL_TYPE_STRING: {
/*
* log_event.h : The first byte is always
* MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the
@@ -494,8 +438,7 @@ public final class TableMapLogEvent extends LogEvent
*/
info.meta = buffer.getUint16();
break;
case MYSQL_TYPE_NEWDECIMAL:
{
case MYSQL_TYPE_NEWDECIMAL: {
int x = buffer.getUint8() << 8; // precision
x += buffer.getUint8(); // decimals
info.meta = x;
@@ -503,8 +446,7 @@ public final class TableMapLogEvent extends LogEvent
}
case MYSQL_TYPE_TIME2:
case MYSQL_TYPE_DATETIME2:
case MYSQL_TYPE_TIMESTAMP2:
{
case MYSQL_TYPE_TIMESTAMP2: {
info.meta = buffer.getUint8();
break;
}
@@ -516,28 +458,23 @@ public final class TableMapLogEvent extends LogEvent
buffer.limit(limit);
}
public final String getDbName()
{
public final String getDbName() {
return dbname;
}
public final String getTableName()
{
public final String getTableName() {
return tblname;
}
public final int getColumnCnt()
{
public final int getColumnCnt() {
return columnCnt;
}
public final ColumnInfo[] getColumnInfo()
{
public final ColumnInfo[] getColumnInfo() {
return columnInfo;
}
public final long getTableId()
{
public final long getTableId() {
return tableId;
}
}

7
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java
View File

@@ -8,10 +8,9 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class UnknownLogEvent extends LogEvent
{
public UnknownLogEvent(LogHeader header)
{
public final class UnknownLogEvent extends LogEvent {
public UnknownLogEvent(LogHeader header){
super(header);
}
}

15
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java
View File

@@ -4,19 +4,16 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
/**
* Log row updates with a before image. The event contain several update rows
* for a table. Note that each event contains only rows for one table.
*
* Also note that the row data consists of pairs of row data: one row for the
* old data and one row for the new data.
* for a table. Note that each event contains only rows for one table. Also note
* that the row data consists of pairs of row data: one row for the old data and
* one row for the new data.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class UpdateRowsLogEvent extends RowsLogEvent
{
public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class UpdateRowsLogEvent extends RowsLogEvent {
public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

59
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java
View File

@@ -8,22 +8,19 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* User_var_log_event.
*
* Every time a query uses the value of a user variable, a User_var_log_event is
* written before the Query_log_event, to set the user variable.
* User_var_log_event. Every time a query uses the value of a user variable, a
* User_var_log_event is written before the Query_log_event, to set the user
* variable.
*
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class UserVarLogEvent extends LogEvent
{
public final class UserVarLogEvent extends LogEvent {
/**
* Fixed data part: Empty
*
* <p>
* Variable data part:
*
* <ul>
* <li>4 bytes. the size of the user variable name.</li>
* <li>The user variable name.</li>
@@ -39,7 +36,6 @@ public final class UserVarLogEvent extends LogEvent
* <li>Variable-sized. For a string variable, this is the string. For a
* float or integer variable, this is its value in 8 bytes.</li>
* </ul>
*
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
*/
private final String name;
@@ -66,46 +62,37 @@ public final class UserVarLogEvent extends LogEvent
public static final int UV_NAME_LEN_SIZE = 4;
public static final int UV_CHARSET_NUMBER_SIZE = 4;
public UserVarLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent) throws IOException
{
public UserVarLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
throws IOException{
super(header);
/* The Post-Header is empty. The Variable Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
final int nameLen = (int) buffer.getUint32();
name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE
isNull = (0 != buffer.getInt8());
if (isNull)
{
if (isNull) {
type = STRING_RESULT;
charsetNumber = 63; /* binary */
value = null;
}
else
{
} else {
type = buffer.getInt8(); // UV_VAL_IS_NULL
charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE
final int valueLen = (int) buffer.getUint32(); // buf + UV_CHARSET_NUMBER_SIZE
final int valueLen = (int) buffer.getUint32(); // buf +
// UV_CHARSET_NUMBER_SIZE
final int limit = buffer.limit(); /* for restore */
buffer.limit(buffer.position() + valueLen);
/* @see User_var_log_event::print */
switch (type)
{
switch (type) {
case REAL_RESULT:
value = Double.valueOf(buffer.getDouble64()); // float8get
break;
case INT_RESULT:
if (valueLen == 8)
value = Long.valueOf(buffer.getLong64()); // !uint8korr
else if (valueLen == 4)
value = Long.valueOf(buffer.getUint32());
else
throw new IOException("Error INT_RESULT length: "
+ valueLen);
if (valueLen == 8) value = Long.valueOf(buffer.getLong64()); // !uint8korr
else if (valueLen == 4) value = Long.valueOf(buffer.getUint32());
else throw new IOException("Error INT_RESULT length: " + valueLen);
break;
case DECIMAL_RESULT:
final int precision = buffer.getInt8();
@@ -127,19 +114,13 @@ public final class UserVarLogEvent extends LogEvent
}
}
public final String getQuery()
{
if (value == null)
{
public final String getQuery() {
if (value == null) {
return "SET @" + name + " := NULL";
}
else if (type == STRING_RESULT)
{
} else if (type == STRING_RESULT) {
// TODO: do escaping !?
return "SET @" + name + " := \'" + value + '\'';
}
else
{
} else {
return "SET @" + name + " := " + String.valueOf(value);
}
}

8
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java
View File

@@ -9,11 +9,9 @@ import com.taobao.tddl.dbsync.binlog.LogBuffer;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class WriteRowsLogEvent extends RowsLogEvent
{
public WriteRowsLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public final class WriteRowsLogEvent extends RowsLogEvent {
public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header, buffer, descriptionEvent);
}
}

14
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java
View File

@@ -10,23 +10,19 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
* @version 1.0
*/
public final class XidLogEvent extends LogEvent
{
public final class XidLogEvent extends LogEvent {
private final long xid;
public XidLogEvent(LogHeader header, LogBuffer buffer,
FormatDescriptionLogEvent descriptionEvent)
{
public XidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
super(header);
/* The Post-Header is empty. The Variable Data part begins immediately. */
buffer.position(descriptionEvent.commonHeaderLen
+ descriptionEvent.postHeaderLen[XID_EVENT - 1]);
buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
xid = buffer.getLong64(); // !uint8korr
}
public final long getXid()
{
public final long getXid() {
return xid;
}
}

24
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java
View File

@@ -10,8 +10,8 @@ import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
public class BaseLogFetcherTest {
@@ -20,36 +20,38 @@ public class BaseLogFetcherTest {
protected Charset charset = Charset.forName("utf-8");
protected void parseQueryEvent(QueryLogEvent event) {
System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
System.out.println(String.format("================> binlog[%s:%s] , name[%s]",
binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getCatalog()));
System.out.println("sql : " + event.getQuery());
}
protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
.getLogPos() - event.getHeader().getEventLen()));
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
}
protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
.getLogPos() - event.getHeader().getEventLen()));
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
}
protected void parseXidEvent(XidLogEvent event) throws Exception {
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, event.getHeader()
.getLogPos() - event.getHeader().getEventLen()));
System.out.println("xid : " + event.getXid());
}
protected void parseRowsEvent(RowsLogEvent event) {
try {
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
binlogFileName,
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
event.getTable().getDbName(), event.getTable().getTableName()));
event.getTable().getDbName(),
event.getTable().getTableName()));
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getChangeColumns();

3
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java
View File

@@ -48,8 +48,7 @@ public class FileLogFetcherTest extends BaseLogFetcherTest {
int eventType = event.getHeader().getType();
switch (eventType) {
case LogEvent.ROTATE_EVENT:
binlogFileName = ((RotateLogEvent)
event).getFilename();
binlogFileName = ((RotateLogEvent) event).getFilename();
break;
case LogEvent.WRITE_ROWS_EVENT_V1:
case LogEvent.WRITE_ROWS_EVENT:

142
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java
View File

@@ -5,14 +5,11 @@ import java.math.BigInteger;
import junit.framework.TestCase;
import com.taobao.tddl.dbsync.binlog.LogBuffer;
public class LogBufferTest extends TestCase {
public class LogBufferTest extends TestCase
{
public static final int LOOP = 10000;
public void testSigned()
{
public void testSigned() {
byte[] array = { 0, 0, 0, (byte) 0xff };
LogBuffer buffer = new LogBuffer(array, 0, array.length);
@@ -24,17 +21,15 @@ public class LogBufferTest extends TestCase
System.out.println(buffer.getUint24(1));
}
public void testBigInteger()
{
byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
public void testBigInteger() {
byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff };
LogBuffer buffer = new LogBuffer(array, 0, array.length);
long tt1 = 0;
long l1 = 0;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t1 = System.nanoTime();
l1 = buffer.getLong64(0);
tt1 += System.nanoTime() - t1;
@@ -45,8 +40,7 @@ public class LogBufferTest extends TestCase
long tt2 = 0;
BigInteger l2 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t2 = System.nanoTime();
l2 = buffer.getUlong64(0);
tt2 += System.nanoTime() - t2;
@@ -57,18 +51,15 @@ public class LogBufferTest extends TestCase
}
/* Reads big-endian integer from no more than 4 bytes */
private static int convertNBytesToInt(byte[] buffer, int offset, int length)
{
private static int convertNBytesToInt(byte[] buffer, int offset, int length) {
int ret = 0;
for (int i = offset; i < (offset + length); i++)
{
for (int i = offset; i < (offset + length); i++) {
ret = (ret << 8) | (0xff & buffer[i]);
}
return ret;
}
private static int convert4BytesToInt(byte[] buffer, int offset)
{
private static int convert4BytesToInt(byte[] buffer, int offset) {
int value;
value = (0xff & buffer[offset + 3]);
value += (0xff & buffer[offset + 2]) << 8;
@@ -77,24 +68,20 @@ public class LogBufferTest extends TestCase
return value;
}
public static short convert1ByteToShort(byte[] buffer, int offset)
{
public static short convert1ByteToShort(byte[] buffer, int offset) {
short value;
value = (short) buffer[offset + 0];
return value;
}
public static short convert2bytesToShort(byte[] buffer, int offset)
{
public static short convert2bytesToShort(byte[] buffer, int offset) {
short value;
value = (short) (buffer[offset + 0] << 8);
value += (short) (buffer[offset + 1] & 0xff);
return value;
}
public static final BigDecimal extractDecimal(byte[] buffer, int precision,
int scale)
{
public static final BigDecimal extractDecimal(byte[] buffer, int precision, int scale) {
//
// Decimal representation in binlog seems to be as follows:
// 1 byte - 'precision'
@@ -129,34 +116,28 @@ public class LogBufferTest extends TestCase
int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
// how many bytes are used to represent given amount of digits?
int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
+ LogBuffer.dig2bytes[intg0x];
int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
+ LogBuffer.dig2bytes[frac0x];
int integerSize = intg0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[intg0x];
int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32 + LogBuffer.dig2bytes[frac0x];
int bin_size = integerSize + decimalSize; // total bytes
byte[] d_copy = new byte[bin_size];
if (bin_size > buffer.length)
{
throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
+ bin_size + ", available bytes: " + buffer.length);
if (bin_size > buffer.length) {
throw new ArrayIndexOutOfBoundsException("Calculated bin_size: " + bin_size + ", available bytes: "
+ buffer.length);
}
// Invert first bit
d_copy[0] = buffer[0];
d_copy[0] ^= 0x80;
if (sign == -1)
{
if (sign == -1) {
// Invert every byte
d_copy[0] ^= 0xFF;
}
for (int i = 1; i < bin_size; i++)
{
for (int i = 1; i < bin_size; i++) {
d_copy[i] = buffer[i];
if (sign == -1)
{
if (sign == -1) {
// Invert every byte
d_copy[i] ^= 0xFF;
}
@@ -167,62 +148,49 @@ public class LogBufferTest extends TestCase
BigDecimal intPart = new BigDecimal(0);
if (offset > 0)
intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
if (offset > 0) intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
while (offset < integerSize)
{
intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
while (offset < integerSize) {
intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(BigDecimal.valueOf(convert4BytesToInt(d_copy,
offset)));
offset += 4;
}
// Decimal part
BigDecimal fracPart = new BigDecimal(0);
int shift = 0;
for (int i = 0; i < frac0; i++)
{
for (int i = 0; i < frac0; i++) {
shift += LogBuffer.DIG_PER_DEC1;
fracPart = fracPart.add(BigDecimal.valueOf(
convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
fracPart = fracPart.add(BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
offset += 4;
}
if (LogBuffer.dig2bytes[frac0x] > 0)
{
fracPart = fracPart.add(BigDecimal.valueOf(
convertNBytesToInt(d_copy, offset,
LogBuffer.dig2bytes[frac0x])).movePointLeft(
shift + frac0x));
if (LogBuffer.dig2bytes[frac0x] > 0) {
fracPart = fracPart.add(BigDecimal.valueOf(convertNBytesToInt(d_copy, offset, LogBuffer.dig2bytes[frac0x]))
.movePointLeft(shift + frac0x));
}
return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
}
public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
0x38, (byte) 0xb0, 0x60, 0x00 };
public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b, 0x38, (byte) 0xb0, 0x60, 0x00 };
public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
(byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
(byte) 0xa0, (byte) 0xff };
public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7,
(byte) 0x4f, (byte) 0xa0, (byte) 0xff };
public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
-108 };
public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96, -108 };
public void testBigDecimal() throws InterruptedException
{
do
{
public void testBigDecimal() throws InterruptedException {
do {
System.out.println("old extract decimal: ");
long tt1 = 0;
BigDecimal bd1 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t1 = System.nanoTime();
bd1 = extractDecimal(array2, 19, 10);
tt1 += System.nanoTime() - t1;
@@ -233,8 +201,7 @@ public class LogBufferTest extends TestCase
long tt2 = 0;
BigDecimal bd2 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t2 = System.nanoTime();
bd2 = extractDecimal(array1, 19, 10);
tt2 += System.nanoTime() - t2;
@@ -245,8 +212,7 @@ public class LogBufferTest extends TestCase
long tt3 = 0;
BigDecimal bd3 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t3 = System.nanoTime();
bd3 = extractDecimal(array3, 18, 6);
tt3 += System.nanoTime() - t3;
@@ -257,8 +223,7 @@ public class LogBufferTest extends TestCase
long tt4 = 0;
BigDecimal bd4 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t4 = System.nanoTime();
bd4 = extractDecimal(array4, 18, 6);
tt4 += System.nanoTime() - t4;
@@ -269,8 +234,7 @@ public class LogBufferTest extends TestCase
long tt5 = 0;
BigDecimal bd5 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t5 = System.nanoTime();
bd5 = extractDecimal(array5, 18, 6);
tt5 += System.nanoTime() - t5;
@@ -278,11 +242,9 @@ public class LogBufferTest extends TestCase
System.out.print(tt5 / LOOP);
System.out.print("ns >> ");
System.out.println(bd5);
}
while (false);
} while (false);
do
{
do {
System.out.println("new extract decimal: ");
LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
@@ -293,8 +255,7 @@ public class LogBufferTest extends TestCase
long tt1 = 0;
BigDecimal bd1 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t1 = System.nanoTime();
bd1 = buffer1.getDecimal(0, 19, 10);
tt1 += System.nanoTime() - t1;
@@ -305,8 +266,7 @@ public class LogBufferTest extends TestCase
long tt2 = 0;
BigDecimal bd2 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t2 = System.nanoTime();
bd2 = buffer2.getDecimal(0, 19, 10);
tt2 += System.nanoTime() - t2;
@@ -317,8 +277,7 @@ public class LogBufferTest extends TestCase
long tt3 = 0;
BigDecimal bd3 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t3 = System.nanoTime();
bd3 = buffer3.getDecimal(0, 18, 6);
tt3 += System.nanoTime() - t3;
@@ -329,8 +288,7 @@ public class LogBufferTest extends TestCase
long tt4 = 0;
BigDecimal bd4 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t4 = System.nanoTime();
bd4 = buffer4.getDecimal(0, 18, 6);
tt4 += System.nanoTime() - t4;
@@ -341,8 +299,7 @@ public class LogBufferTest extends TestCase
long tt5 = 0;
BigDecimal bd5 = null;
for (int i = 0; i < LOOP; i++)
{
for (int i = 0; i < LOOP; i++) {
final long t5 = System.nanoTime();
bd5 = buffer5.getDecimal(0, 18, 6);
tt5 += System.nanoTime() - t5;
@@ -350,7 +307,6 @@ public class LogBufferTest extends TestCase
System.out.print(tt5 / LOOP);
System.out.print("ns >> ");
System.out.println(bd5);
}
while (false);
} while (false);
}
}

93
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.deployer;
import com.google.common.collect.MigrateMap;
import java.util.Map;
import java.util.Properties;
@@ -39,6 +38,7 @@ import com.alibaba.otter.canal.server.exception.CanalServerException;
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* canal调度控制器
@@ -71,11 +71,9 @@ public class CanalController {
}
public CanalController(final Properties properties){
managerClients = MigrateMap.makeComputingMap(new Function<String, CanalConfigClient>()
{
managerClients = MigrateMap.makeComputingMap(new Function<String, CanalConfigClient>() {
public CanalConfigClient apply(String managerAddress)
{
public CanalConfigClient apply(String managerAddress) {
return getManagerClient(managerAddress);
}
});
@@ -110,88 +108,68 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>()
{
public ServerRunningMonitor apply(final String destination)
{
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener()
{
runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter()
{
try
{
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
} finally
{
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processActiveExit()
{
try
{
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally
{
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart()
{
try
{
if (zkclientx != null)
{
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener()
{
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception
{
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception
{
public void handleNewSession() throws Exception {
initCid(path);
}
});
}
} finally
{
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop()
{
try
{
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null)
{
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
releaseCid(path);
}
} finally
{
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null)
{
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
return runningMonitor;
@@ -239,32 +217,25 @@ public class CanalController {
}
};
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>()
{
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
public InstanceConfigMonitor apply(InstanceMode mode)
{
int scanInterval = Integer
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring())
{
if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
// 设置conf目录，默认是user.dir + conf目录组成
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir))
{
if (StringUtils.isEmpty(rootDir)) {
rootDir = "../conf";
}
monitor.setRootConf(rootDir);
return monitor;
} else if (mode.isManager())
{
} else if (mode.isManager()) {
return new ManagerInstanceConfigMonitor();
} else
{
} else {
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
}
}

8
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.deployer.monitor;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.FileFilter;
import java.io.FilenameFilter;
@@ -25,6 +24,7 @@ import com.alibaba.otter.canal.common.CanalLifeCycle;
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 监听基于spring配置的instance变化
@@ -40,11 +40,9 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>()
{
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>() {
public InstanceConfigFiles apply(String destination)
{
public InstanceConfigFiles apply(String destination) {
return new InstanceConfigFiles(destination);
}
});

8
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java
View File

@@ -14,15 +14,17 @@ import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
* </pre>
*
* <br>
* The Packet Header will not be shown in the descriptions of packets that follow this section. Think of it as always
* there. But logically, it "precedes the packet" rather than "is included in the packet".<br>
* The Packet Header will not be shown in the descriptions of packets that
* follow this section. Think of it as always there. But logically, it
* "precedes the packet" rather than "is included in the packet".<br>
*
* @author fujohnwang
*/
public class HeaderPacket implements IPacket {
/**
* this field indicates the packet length that follows the header, with header packet's 4 bytes excluded.
* this field indicates the packet length that follows the header, with
* header packet's 4 bytes excluded.
*/
private int packetBodyLength;
private byte packetSequenceNumber;

1
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java
View File

@@ -12,6 +12,7 @@ import java.io.IOException;
* @see 1.0
*/
public interface IPacket {
/**
* un-marshal raw bytes into {@link IPacket} state for application usage.<br>
*

3
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/HandshakeInitializationPacket.java
View File

@@ -75,7 +75,8 @@ public class HandshakeInitializationPacket extends PacketWithHeaderPacket {
// 8. bypass filtered bytes
index += 13;
// 9. read rest of scramble_buff
this.restOfScrambleBuff = ByteHelper.readFixedLengthBytes(data, index, 12); // 虽然Handshake Initialization
this.restOfScrambleBuff = ByteHelper.readFixedLengthBytes(data, index, 12); // 虽然Handshake
// Initialization
// Packet规定最后13个byte是剩下的scrumble,
// 但实际上最后一个字节是0, 不应该包含在scrumble中.
// end read

6
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java
View File

@@ -30,8 +30,10 @@ public abstract class PacketManager {
}
/**
* Since We r using blocking IO, so we will just write once and assert the length to simplify the read operation.<br>
* If the block write doesn't work as we expected, we will change this implementation as per the result.
* Since We r using blocking IO, so we will just write once and assert the
* length to simplify the read operation.<br>
* If the block write doesn't work as we expected, we will change this
* implementation as per the result.
*
* @param ch
* @param len

40
filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
View File

@@ -1,14 +1,16 @@
package com.alibaba.otter.canal.filter;
import java.util.Map;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
import java.util.Map;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler;
/**
* 提供{@linkplain Pattern}的lazy get处理
@@ -16,39 +18,29 @@ import org.apache.oro.text.regex.Perl5Compiler;
* @author jianghang 2013-1-22 下午09:36:44
* @version 1.0.0
*/
public class PatternUtils
{
public class PatternUtils {
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, Pattern>()
{
new Function<String, Pattern>() {
public Pattern apply(
String pattern)
{
try
{
public Pattern apply(String pattern) {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(
pattern,
return pc.compile(pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK
| Perl5Compiler.READ_ONLY_MASK
| Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e)
{
throw new CanalFilterException(
e);
} catch (MalformedPatternException e) {
throw new CanalFilterException(e);
}
}
});
public static Pattern getPattern(String pattern)
{
public static Pattern getPattern(String pattern) {
return patterns.get(pattern);
}
public static void clear()
{
public static void clear() {
patterns.clear();
}
}

6
filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java
View File

@@ -67,16 +67,14 @@ public class AviaterFilterTest {
result = filter3.filter("fooooot");
Assert.assertEquals(true, result);
AviaterRegexFilter filter4 = new AviaterRegexFilter(
"otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
AviaterRegexFilter filter4 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
result = filter4.filter("otter1.otter_stability1");
Assert.assertEquals(true, result);
}
@Test
public void testDisordered() {
AviaterRegexFilter filter = new AviaterRegexFilter(
"u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");
AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");
boolean result = filter.filter("u.abc");
Assert.assertEquals(true, result);

3
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
View File

@@ -115,8 +115,7 @@ public class CanalInstanceWithManager extends CanalInstanceSupport implements Ca
public void start() {
super.start();
// 初始化metaManager
logger.info("start CannalInstance for {}-{} with parameters:{}", canalId, destination,
parameters);
logger.info("start CannalInstance for {}-{} with parameters:{}", canalId, destination, parameters);
if (!metaManager.isStart()) {
metaManager.start();

6
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java
View File

@@ -11,8 +11,10 @@ import org.springframework.core.io.ResourceLoader;
import org.springframework.util.Assert;
/**
* 扩展Spring的{@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer} ，增加默认值的功能。
* 例如：${placeholder:defaultValue}，假如placeholder的值不存在，则默认取得 defaultValue。
* 扩展Spring的
* {@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer}
* ，增加默认值的功能。 例如：${placeholder:defaultValue}，假如placeholder的值不存在，则默认取得
* defaultValue。
*
* @author jianghang 2013-1-24 下午03:37:56
* @version 1.0.0

36
meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -27,7 +26,7 @@ import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 基于文件刷新的metaManager实现
@@ -70,36 +69,28 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>()
{
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination)
{
public File apply(String destination) {
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return loadClientIdentity(destination);
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity)
{
public Position apply(ClientIdentity clientIdentity) {
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null)
{
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else
{
} else {
return position;
}
}
@@ -118,8 +109,8 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
// 定时将内存中的最新值刷到file中，多次变更只刷一次
if (logger.isInfoEnabled()) {
LogPosition cursor = (LogPosition) getCursor(clientIdentity);
logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] {
clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
logger.info("clientId:{} cursor:[{},{},{}] address[{}]",
new Object[] { clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getIdentity().getSourceAddress().toString() });
}
@@ -131,7 +122,10 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
}
}
}
}, period, period, TimeUnit.MILLISECONDS);
},
period,
period,
TimeUnit.MILLISECONDS);
}
public void stop() {

17
meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -16,6 +15,7 @@ import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.common.collect.MigrateMap;
/**
* 内存版实现
@@ -32,11 +32,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
public void start() {
super.start();
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
return MemoryClientIdentityBatch.create(clientIdentity);
}
@@ -44,11 +42,9 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
cursors = new MapMaker().makeMap();
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return Lists.newArrayList();
}
});
@@ -167,7 +163,8 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
Long minBatchId = Collections.min(batches.keySet());
if (!minBatchId.equals(batchId)) {
// 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId,
throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d",
batchId,
minBatchId));
}
return batches.remove(batchId);

132
meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java
View File

@@ -1,16 +1,18 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
/**
* 组合memory + zookeeper的使用模式
@@ -19,61 +21,48 @@ import org.springframework.util.Assert;
* @version 1.0.0
*/
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager
{
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private ExecutorService executor;
private ZooKeeperMetaManager zooKeeperMetaManager;
@SuppressWarnings("serial")
private final Position nullCursor = new Position()
{
private final Position nullCursor = new Position() {
};
public void start()
{
public void start() {
super.start();
Assert.notNull(zooKeeperMetaManager);
if (!zooKeeperMetaManager.isStart())
{
if (!zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.start();
}
executor = Executors.newFixedThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity)
{
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null)
{
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else
{
} else {
return position;
}
}
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
{
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
@@ -81,12 +70,10 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
});
}
public void stop()
{
public void stop() {
super.stop();
if (zooKeeperMetaManager.isStart())
{
if (zooKeeperMetaManager.isStart()) {
zooKeeperMetaManager.stop();
}
@@ -95,73 +82,58 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
batches.clear();
}
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity);
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
}
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
super.updateCursor(clientIdentity, position);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.updateCursor(clientIdentity, position);
}
});
}
@Override
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
Position position = super.getCursor(clientIdentity);
if (position == nullCursor)
{
if (position == nullCursor) {
return null;
} else
{
} else {
return position;
}
}
public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
final Long batchId = super.addBatch(clientIdentity, positionRange);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
@@ -169,30 +141,24 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
}
public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
super.addBatch(clientIdentity, positionRange, batchId);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
}
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
throws CanalMetaManagerException
{
throws CanalMetaManagerException {
PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
}
});
@@ -200,24 +166,20 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
return positionRange;
}
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.clearAllBatchs(clientIdentity);
// 异步刷新
executor.submit(new Runnable()
{
executor.submit(new Runnable() {
public void run()
{
public void run() {
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
}
});
}
// =============== setter / getter ================
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager)
{
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
this.zooKeeperMetaManager = zooKeeperMetaManager;
}
}

31
meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -20,7 +19,7 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 基于定时刷新的策略的mixed实现
@@ -53,40 +52,32 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
}
executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination)
{
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity)
{
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null)
{
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else
{
} else {
return position;
}
}
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
{
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;

21
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
import com.google.common.collect.MigrateMap;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -16,7 +15,7 @@ import com.alibaba.otter.canal.parse.inbound.TableMeta;
import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 处理table meta解析和缓存
@@ -39,23 +38,17 @@ public class TableMetaCache {
public TableMetaCache(MysqlConnection con){
this.connection = con;
tableMetaCache = MigrateMap.makeComputingMap(new Function<String, TableMeta>()
{
tableMetaCache = MigrateMap.makeComputingMap(new Function<String, TableMeta>() {
public TableMeta apply(String name)
{
try
{
public TableMeta apply(String name) {
try {
return getTableMeta0(name);
} catch (IOException e)
{
} catch (IOException e) {
// 尝试做一次retry操作
try
{
try {
connection.reconnect();
return getTableMeta0(name);
} catch (IOException e1)
{
} catch (IOException e1) {
throw new CanalParseException("fetch failed by table meta:" + name, e1);
}
}

21
parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.parse.index;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -23,7 +22,7 @@ import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 基于文件刷新的log position实现
@@ -68,27 +67,21 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>()
{
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination)
{
public File apply(String destination) {
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>()
{
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {
public LogPosition apply(String destination)
{
public LogPosition apply(String destination) {
LogPosition logPosition = loadDataFromFile(dataFileCaches.get(destination));
if (logPosition == null)
{
if (logPosition == null) {
return nullPosition;
} else
{
} else {
return logPosition;
}
}

15
parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.parse.index;
import com.google.common.collect.MigrateMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -10,7 +9,7 @@ import org.springframework.util.Assert;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 混合memory + zookeeper的存储模式
@@ -35,17 +34,13 @@ public class MixedLogPositionManager extends MemoryLogPositionManager implements
zooKeeperLogPositionManager.start();
}
executor = Executors.newFixedThreadPool(1);
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>()
{
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {
public LogPosition apply(String destination)
{
public LogPosition apply(String destination) {
LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
if (logPosition == null)
{
if (logPosition == null) {
return nullPosition;
} else
{
} else {
return logPosition;
}
}

15
parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.parse.index;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -16,7 +15,7 @@ import org.springframework.util.Assert;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
/**
* 基于定时刷新的策略的mixed实现
@@ -44,17 +43,13 @@ public class PeriodMixedLogPositionManager extends MemoryLogPositionManager impl
zooKeeperLogPositionManager.start();
}
executor = Executors.newScheduledThreadPool(1);
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>()
{
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {
public LogPosition apply(String destination)
{
public LogPosition apply(String destination) {
LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
if (logPosition == null)
{
if (logPosition == null) {
return nullPosition;
} else
{
} else {
return logPosition;
}
}

6
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java
View File

@@ -86,9 +86,11 @@ public class EventTransactionBufferTest {
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] {
// System.out.println(MessageFormat.format(messgae, new
// Object[] {
// Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntryType());

36
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java
View File

@@ -87,17 +87,17 @@ public class DummyEventStore implements CanalEventStore<Event> {
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(),
header.getLogfileName(),
String.valueOf(header.getLogfileOffset()),
format.format(date), header.getEventType(),
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
}
}
@@ -110,17 +110,17 @@ public class DummyEventStore implements CanalEventStore<Event> {
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(),
header.getLogfileName(),
String.valueOf(header.getLogfileOffset()),
format.format(date), header.getEventType(),
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
}
}
@@ -136,17 +136,17 @@ public class DummyEventStore implements CanalEventStore<Event> {
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
|| data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
// System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(), format.format(date),
// System.out.println(MessageFormat.format(messgae, new Object[]
// { Thread.currentThread().getName(),
// header.getLogfilename(), header.getLogfileoffset(),
// format.format(date),
// data.getEntry().getEntryType(), "" }));
System.out.println(data.getEntry().getEntryType());
} else {
System.out.println(MessageFormat.format(messgae,
new Object[] { Thread.currentThread().getName(),
header.getLogfileName(),
String.valueOf(header.getLogfileOffset()),
format.format(date), header.getEventType(),
new Object[] { Thread.currentThread().getName(), header.getLogfileName(),
String.valueOf(header.getLogfileOffset()), format.format(date), header.getEventType(),
header.getSchemaName(), header.getTableName() }));
}

4
protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java
View File

@@ -1,6 +1,7 @@
package com.alibaba.otter.canal.protocol;
import java.io.Serializable;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
@@ -10,8 +11,7 @@ import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
* @author zebin.xuzb @ 2012-6-20
* @version 1.0.0
*/
public class ClientIdentity implements Serializable
{
public class ClientIdentity implements Serializable {
private String destination;
private short clientId;

10
server/src/main/java/com/alibaba/otter/canal/server/CanalService.java
View File

@@ -1,20 +1,20 @@
package com.alibaba.otter.canal.server;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.exception.CanalServerException;
import java.util.concurrent.TimeUnit;
public interface CanalService
{
public interface CanalService {
void subscribe(ClientIdentity clientIdentity) throws CanalServerException;
void unsubscribe(ClientIdentity clientIdentity) throws CanalServerException;
Message get(ClientIdentity clientIdentity, int batchSize) throws CanalServerException;
Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
throws CanalServerException;
Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException;
Message getWithoutAck(ClientIdentity clientIdentity, int batchSize) throws CanalServerException;

47
server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
View File

@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.server.embedded;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -29,6 +28,7 @@ import com.alibaba.otter.canal.store.model.Events;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.MigrateMap;
/**
* 嵌入式版本实现
@@ -37,8 +37,7 @@ import com.google.common.collect.Maps;
* @author zebin.xuzb
* @version 1.0.0
*/
public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, com.alibaba.otter.canal.server.CanalService
{
public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, com.alibaba.otter.canal.server.CanalService {
private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class);
private Map<String, CanalInstance> canalInstances;
@@ -48,11 +47,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
public void start() {
super.start();
canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>()
{
canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>() {
public CanalInstance apply(String destination)
{
public CanalInstance apply(String destination) {
return canalInstanceGenerator.generate(destination);
}
});
@@ -195,9 +192,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
if (positionRanges != null) {
throw new CanalServerException(
String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data",
clientIdentity.getClientId(), positionRanges));
throw new CanalServerException(String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data",
clientIdentity.getClientId(),
positionRanges));
}
Events<Event> events = null;
@@ -219,7 +216,10 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
});
logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
clientIdentity.getClientId(), batchSize, entrys.size(), batchId,
clientIdentity.getClientId(),
batchSize,
entrys.size(),
batchId,
events.getPositionRange());
// 直接提交ack
ack(clientIdentity, batchId);
@@ -293,7 +293,10 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
});
logger.info("getWithoutAck successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
clientIdentity.getClientId(), batchSize, entrys.size(), batchId,
clientIdentity.getClientId(),
batchSize,
entrys.size(),
batchId,
events.getPositionRange());
return new Message(batchId, entrys);
}
@@ -331,9 +334,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
PositionRange<LogPosition> positionRanges = null;
positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId); // 更新位置
if (positionRanges == null) { // 说明是重复的ack/rollback
throw new CanalServerException(
String.format("ack error , clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(), batchId));
throw new CanalServerException(String.format("ack error , clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(),
batchId));
}
// 更新cursor最好严格判断下位置是否有跳跃更新
@@ -357,7 +360,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
if (positionRanges.getAck() != null) {
canalInstance.getMetaManager().updateCursor(clientIdentity, positionRanges.getAck());
logger.info("ack successfully, clientId:{} batchId:{} position:{}",
clientIdentity.getClientId(), batchId, positionRanges);
clientIdentity.getClientId(),
batchId,
positionRanges);
}
// 可定时清理数据
@@ -405,9 +410,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
batchId);
if (positionRanges == null) { // 说明是重复的ack/rollback
throw new CanalServerException(
String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(), batchId));
throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(),
batchId));
}
// lastRollbackPostions.put(clientIdentity,
@@ -416,7 +421,9 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
canalInstance.getEventStore().rollback();// rollback
// eventStore中的状态信息
logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
clientIdentity.getClientId(), batchId, positionRanges);
clientIdentity.getClientId(),
batchId,
positionRanges);
}
}

20
server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
View File

@@ -32,9 +32,12 @@ public class NettyUtils {
}
public static void ack(Channel channel, ChannelFutureListener channelFutureListner) {
write(
channel,
Packet.newBuilder().setType(CanalPacket.PacketType.ACK).setBody(Ack.newBuilder().build().toByteString()).build().toByteArray(),
write(channel,
Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray(),
channelFutureListner);
}
@@ -45,11 +48,12 @@ public class NettyUtils {
}
logger.error("ErrotCode:{} , Caused by : \n{}", errorCode, errorMessage);
write(
channel,
Packet.newBuilder().setType(CanalPacket.PacketType.ACK).setBody(
Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(
errorMessage).build().toByteString()).build().toByteArray(),
write(channel,
Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray(),
channelFutureListener);
}
}

3
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java
View File

@@ -62,8 +62,7 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
ctx.setAttachment(clientIdentity);// 设置状态数据
// 尝试启动，如果已经启动，忽略
if (!embeddedServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity
.getDestination());
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (!runningMonitor.isStart()) {
runningMonitor.start();
}

7
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java
View File

@@ -22,8 +22,11 @@ public class HandshakeInitializationHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(HandshakeInitializationHandler.class);
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
byte[] body = Packet.newBuilder().setType(CanalPacket.PacketType.HANDSHAKE).setBody(
Handshake.newBuilder().build().toByteString()).build().toByteArray();
byte[] body = Packet.newBuilder()
.setType(CanalPacket.PacketType.HANDSHAKE)
.setBody(Handshake.newBuilder().build().toByteString())
.build()
.toByteArray();
NettyUtils.write(ctx.getChannel(), body, null);
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}

54
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
View File

@@ -45,7 +45,8 @@ public class SessionHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class);
private CanalServerWithEmbedded embeddedServer;
public SessionHandler(){ }
public SessionHandler(){
}
public SessionHandler(CanalServerWithEmbedded embeddedServer){
this.embeddedServer = embeddedServer;
@@ -61,7 +62,8 @@ public class SessionHandler extends SimpleChannelHandler {
case SUBSCRIPTION:
Sub sub = Sub.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()),
clientIdentity = new ClientIdentity(sub.getDestination(),
Short.valueOf(sub.getClientId()),
sub.getFilter());
MDC.put("destination", clientIdentity.getDestination());
embeddedServer.subscribe(clientIdentity);
@@ -79,13 +81,15 @@ public class SessionHandler extends SimpleChannelHandler {
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage(),
ctx.getChannel(), null);
ctx.getChannel(),
null);
}
break;
case UNSUBSCRIPTION:
Unsub unsub = Unsub.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()),
clientIdentity = new ClientIdentity(unsub.getDestination(),
Short.valueOf(unsub.getClientId()),
unsub.getFilter());
MDC.put("destination", clientIdentity.getDestination());
embeddedServer.unsubscribe(clientIdentity);
@@ -94,7 +98,8 @@ public class SessionHandler extends SimpleChannelHandler {
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage(),
ctx.getChannel(), null);
ctx.getChannel(),
null);
}
break;
case GET:
@@ -106,17 +111,21 @@ public class SessionHandler extends SimpleChannelHandler {
// if (get.getAutoAck()) {
// if (get.getTimeout() == -1) {//是否是初始值
// message = embeddedServer.get(clientIdentity, get.getFetchSize());
// message = embeddedServer.get(clientIdentity,
// get.getFetchSize());
// } else {
// TimeUnit unit = convertTimeUnit(get.getUnit());
// message = embeddedServer.get(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
// message = embeddedServer.get(clientIdentity,
// get.getFetchSize(), get.getTimeout(), unit);
// }
// } else {
if (get.getTimeout() == -1) {// 是否是初始值
message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize());
} else {
TimeUnit unit = convertTimeUnit(get.getUnit());
message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(),
message = embeddedServer.getWithoutAck(clientIdentity,
get.getFetchSize(),
get.getTimeout(),
unit);
}
// }
@@ -136,7 +145,8 @@ public class SessionHandler extends SimpleChannelHandler {
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(),
ctx.getChannel(), null);
ctx.getChannel(),
null);
}
break;
case CLIENTACK:
@@ -146,7 +156,8 @@ public class SessionHandler extends SimpleChannelHandler {
if (ack.getBatchId() == 0L) {
NettyUtils.error(402,
MessageFormatter.format("batchId should assign value", ack.toString()).getMessage(),
ctx.getChannel(), null);
ctx.getChannel(),
null);
} else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据，直接忽略之
// donothing
} else {
@@ -156,7 +167,8 @@ public class SessionHandler extends SimpleChannelHandler {
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage(),
ctx.getChannel(), null);
ctx.getChannel(),
null);
}
break;
case CLIENTROLLBACK:
@@ -173,28 +185,32 @@ public class SessionHandler extends SimpleChannelHandler {
}
} else {
NettyUtils.error(401,
MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage(),
ctx.getChannel(), null);
MessageFormatter.format("destination or clientId is null", rollback.toString())
.getMessage(),
ctx.getChannel(),
null);
}
break;
default:
NettyUtils.error(400,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage(),
ctx.getChannel(), null);
NettyUtils.error(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType())
.getMessage(), ctx.getChannel(), null);
break;
}
} catch (Throwable exception) {
NettyUtils.error(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage(),
ctx.getChannel(), null);
ctx.getChannel(),
ExceptionUtils.getStackTrace(exception)).getMessage(),
ctx.getChannel(),
null);
} finally {
MDC.remove("destination");
}
}
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
logger.error("something goes wrong with channel:{}, exception={}", ctx.getChannel(),
logger.error("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(e.getCause()));
ctx.getChannel().close();

6
server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java
View File

@@ -50,11 +50,9 @@ public class CanalServerWithNettyTest {
@Before
public void setUp() {
CanalServerWithEmbedded embeddedServer = new CanalServerWithEmbedded();
embeddedServer.setCanalInstanceGenerator(new CanalInstanceGenerator()
{
embeddedServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {
public CanalInstance generate(String destination)
{
public CanalInstance generate(String destination) {
Canal canal = buildCanal();
return new CanalInstanceWithManager(canal, FILTER);
}

3
sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java
View File

@@ -132,7 +132,8 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
boolean need = filter.filter(name);
if (!need) {
logger.debug("filter name[{}] entry : {}:{}",
name, event.getEntry().getHeader().getLogfileName(),
name,
event.getEntry().getHeader().getLogfileName(),
event.getEntry().getHeader().getLogfileOffset());
}

6
sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineTransactionBarrier.java
View File

@@ -65,12 +65,14 @@ public class TimelineTransactionBarrier extends TimelineBarrier {
inTransaction.set(false); // 事务结束并且已经成功写入store，清理标记，进入重新排队判断，允许新的事务进入
txState.compareAndSet(1, 0);
// if (txState.compareAndSet(1, 0) == false) {
// throw new CanalSinkException("state is not correct in transaction");
// throw new
// CanalSinkException("state is not correct in transaction");
// }
} else if (txState.intValue() == 2) {// 非事务中
txState.compareAndSet(2, 0);
// if (txState.compareAndSet(2, 0) == false) {
// throw new CanalSinkException("state is not correct in non-transaction");
// throw new
// CanalSinkException("state is not correct in non-transaction");
// }
}
}

8
store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java
View File

@@ -31,7 +31,10 @@ import com.alibaba.otter.canal.store.model.Events;
*/
public class MemoryEventStoreMultiThreadTest extends MemoryEventStoreBase {
private ExecutorService executor = Executors.newFixedThreadPool(2); // 1 producer ,1 cousmer
private ExecutorService executor = Executors.newFixedThreadPool(2); // 1
// producer
// ,1
// cousmer
private MemoryEventStoreWithBuffer eventStore;
@Before
@@ -149,7 +152,8 @@ public class MemoryEventStoreMultiThreadTest extends MemoryEventStoreBase {
try {
Events<Event> entrys = eventStore.get(first, batchSize, 1000L, TimeUnit.MILLISECONDS);
// Events<Event> entrys = eventStore.tryGet(first, batchSize);
// Events<Event> entrys = eventStore.tryGet(first,
// batchSize);
if (!CollectionUtils.isEmpty(entrys.getEvents())) {
if (entrys.getEvents().size() != batchSize) {
System.out.println("get size:" + entrys.getEvents().size() + " with not full batchSize:"