Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Initial commit

  • Loading branch information...
commit 1bed531fae042c52827721d7178b388ad6ab7953 0 parents
@harlowja harlowja authored
Showing with 6,897 additions and 0 deletions.
  1. +270 −0 FORMATTER.xml
  2. +14 −0 HEADER
  3. +32 −0 LICENSE
  4. +131 −0 README
  5. +72 −0 commons/pom.xml
  6. +103 −0 commons/src/main/java/com/yahoo/flowetl/commons/db/Field.java
  7. +369 −0 commons/src/main/java/com/yahoo/flowetl/commons/db/JoinPipe.java
  8. +159 −0 commons/src/main/java/com/yahoo/flowetl/commons/db/SelectPipe.java
  9. +34 −0 commons/src/main/java/com/yahoo/flowetl/commons/db/actions/Action.java
  10. +170 −0 commons/src/main/java/com/yahoo/flowetl/commons/db/actions/ActionSequencer.java
  11. +117 −0 commons/src/main/java/com/yahoo/flowetl/commons/db/actions/UpdateAction.java
  12. +138 −0 commons/src/main/java/com/yahoo/flowetl/commons/runner/BaseRunner.java
  13. +236 −0 commons/src/main/java/com/yahoo/flowetl/commons/runner/CommandLineListener.java
  14. +125 −0 commons/src/main/java/com/yahoo/flowetl/commons/runner/Main.java
  15. +27 −0 commons/src/main/java/com/yahoo/flowetl/commons/runner/Runner.java
  16. +161 −0 commons/src/main/java/com/yahoo/flowetl/commons/web/HttpPipe.java
  17. +56 −0 commons/src/test/java/com/yahoo/flowetl/db/tests/TestJoinPipe.java
  18. +141 −0 commons/src/test/java/com/yahoo/flowetl/web/tests/TestHttpPipe.java
  19. +14 −0 commons/src/test/resources/log4j.properties
  20. +84 −0 core/pom.xml
  21. +56 −0 core/src/main/java/com/yahoo/flowetl/core/CoreException.java
  22. +28 −0 core/src/main/java/com/yahoo/flowetl/core/InputValidator.java
  23. +148 −0 core/src/main/java/com/yahoo/flowetl/core/Logger.java
  24. +54 −0 core/src/main/java/com/yahoo/flowetl/core/PipeException.java
  25. +107 −0 core/src/main/java/com/yahoo/flowetl/core/PipeRunner.java
  26. +339 −0 core/src/main/java/com/yahoo/flowetl/core/Plumber.java
  27. +31 −0 core/src/main/java/com/yahoo/flowetl/core/iterator/CloseableIterator.java
  28. +64 −0 core/src/main/java/com/yahoo/flowetl/core/listener/FlowListener.java
  29. +256 −0 core/src/main/java/com/yahoo/flowetl/core/pipe/AbstractPipe.java
  30. +121 −0 core/src/main/java/com/yahoo/flowetl/core/pipe/Pipe.java
  31. +68 −0 core/src/main/java/com/yahoo/flowetl/core/pipe/PipeResult.java
  32. +68 −0 core/src/main/java/com/yahoo/flowetl/core/pipe/example/CapturePipe.java
  33. +48 −0 core/src/main/java/com/yahoo/flowetl/core/pipe/example/VoidPipe.java
  34. +178 −0 core/src/main/java/com/yahoo/flowetl/core/pipe/result/BackedPipeResult.java
  35. +52 −0 core/src/main/java/com/yahoo/flowetl/core/services/Service.java
  36. +73 −0 core/src/main/java/com/yahoo/flowetl/core/services/ServiceRegistry.java
  37. +80 −0 core/src/main/java/com/yahoo/flowetl/core/util/EnumUtils.java
  38. +84 −0 core/src/main/java/com/yahoo/flowetl/core/util/IterUtils.java
  39. +115 −0 core/src/main/java/com/yahoo/flowetl/core/util/KlassUtils.java
  40. +58 −0 core/src/main/java/com/yahoo/flowetl/core/util/NetUtils.java
  41. +53 −0 core/src/main/java/com/yahoo/flowetl/core/util/Pair.java
  42. +106 −0 core/src/main/java/com/yahoo/flowetl/core/util/RegexUtil.java
  43. +108 −0 core/src/main/java/com/yahoo/flowetl/core/validator/MapInputValidator.java
  44. +56 −0 core/src/test/java/com/yahoo/flowetl/flow/TestFlow.java
  45. +20 −0 core/src/test/java/com/yahoo/flowetl/flow/TestServiceRegistry.java
  46. +14 −0 core/src/test/resources/log4j.properties
  47. +70 −0 distribution/pom.xml
  48. +268 −0 pom.xml
  49. +4 −0 properties/c3p0.properties
  50. +26 −0 properties/log4j.properties
  51. +78 −0 services/pom.xml
  52. +123 −0 services/src/main/java/com/yahoo/flowetl/services/ConfigService.java
  53. +58 −0 services/src/main/java/com/yahoo/flowetl/services/DatabaseService.java
  54. +281 −0 services/src/main/java/com/yahoo/flowetl/services/HttpService.java
  55. +90 −0 services/src/main/java/com/yahoo/flowetl/services/MemoryService.java
  56. +143 −0 services/src/main/java/com/yahoo/flowetl/services/config/CompositeConfigService.java
  57. +94 −0 services/src/main/java/com/yahoo/flowetl/services/config/DummyConfigService.java
  58. +109 −0 services/src/main/java/com/yahoo/flowetl/services/db/CachingDatabaseService.java
  59. +92 −0 services/src/main/java/com/yahoo/flowetl/services/factory/ServiceFactory.java
  60. +179 −0 services/src/main/java/com/yahoo/flowetl/services/http/BaseHttpCaller.java
  61. +191 −0 services/src/main/java/com/yahoo/flowetl/services/http/BaseHttpGenerator.java
  62. +86 −0 services/src/main/java/com/yahoo/flowetl/services/memory/MapMemoryService.java
  63. +27 −0 services/src/test/java/com/yahoo/flowetl/services/tests/TestUtil.java
  64. +122 −0 services/src/test/java/com/yahoo/flowetl/services/tests/mocks/MockHttpService.java
  65. +18 −0 services/src/test/resources/log4j.properties
  66. 0  services/src/test/resources/test.properties
270 FORMATTER.xml
@@ -0,0 +1,270 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<profiles version="11">
+ <!-- For eclipse formatting import so that people can follow my conventions -->
+ <profile kind="CodeFormatterProfile" name="FlowEtl" version="11">
+ <setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="next_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="0"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
+ <setting id="org.eclipse.jdt.core.compiler.source" value="1.5"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="0"/>
+ <setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.lineSplit" value="150"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
+ <setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
+ <setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
+ <setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="next_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="next_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="4"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
+ <setting id="org.eclipse.jdt.core.compiler.compliance" value="1.5"/>
+ <setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="next_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
+ <setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="80"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="insert"/>
+ <setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.5"/>
+ <setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="0"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="next_line"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+ <setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
+ <setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
+ </profile>
+</profiles>
14 HEADER
@@ -0,0 +1,14 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+
+
32 LICENSE
@@ -0,0 +1,32 @@
+Copyright (c) 2011, Yahoo! Inc. All rights reserved.
+(New BSD License)
+
+Redistribution and use of this software in source and binary forms,
+with or without modification, are permitted provided that the following
+conditions are met:
+
+* Redistributions of source code must retain the above
+ copyright notice, this list of conditions and the
+ following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the
+ following disclaimer in the documentation and/or other
+ materials provided with the distribution.
+
+* Neither the name of Yahoo! Inc. nor the names of its
+ contributors may be used to endorse or promote products
+ derived from this software without specific prior
+ written permission of Yahoo! Inc.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
131 README
@@ -0,0 +1,131 @@
+---
+
+This is a framework designed for the creation of testable components which can
+be interconnected via arbitrary inputs and outputs and those components can
+be executed in the correct order (inputs satisfied before running) automatically.
+This is useful since it aids developers in thinking in the paradigm where they plan components
+ahead of time, allowing for simpler reuse and refactoring later. At yahoo! this
+was created for a ETL like framework & pipeline, but its applications are not
+limited to solely that, since the concept itself is generic.
+
+* Future developments
+ - Integrate more services
+ - Multi-threaded planning and execution
+ - More tests!
+
+---
+
+The FlowEtl project uses several 3rd party open source libraries and tools.
+This file summarizes the tools used, their purpose, and the licenses under which
+they're released.
+
+---
+
+Except as specifically stated below, the 3rd party software packages are not
+distributed as part of this project, but instead are separately downloaded from
+the respective provider and built on the developer’s machine as a pre-build
+step.
+
+* commons-io 2.0.1 (Apache license 2.0)
+ - Commons IO is a library of utilities
+ to assist with developing IO functionality.
+ - https://commons.apache.org/io/
+
+* json-simple 1.1 (Apache license 2.0)
+ - JSON.simple is a simple Java toolkit for JSON.
+ You can use JSON.simple to encode or decode JSON text.
+ - https://code.google.com/p/json-simple/
+
+* commons-configuration 1.4 (Apache license 2.0)
+ - Commons Configuration provides a generic configuration
+ interface which enables a Java application
+ to read configuration data from a variety of sources.
+ - https://commons.apache.org/configuration/
+
+* commons-lang 2.6 (Apache license 2.0)
+ - Commons Lang, a package of Java utility classes for
+ the classes that are in java.lang's hierarchy, or are
+ considered to be so standard as to justify existence in java.lang.
+ - https://commons.apache.org/lang/
+
+* commons-httpclient 3.1 (Apache license 2.0)
+ - Commons HttpClient is a library for client-side HTTP communication.
+ - http://hc.apache.org/httpclient-3.x/
+
+* commons-collections 3.2.1 (Apache license 2.0)
+ - Types that extend and augment the Java Collections Framework.
+ - http://commons.apache.org/collections/
+
+* commons-codec 1.4 (Apache license 2.0)
+ - Contains simple encoder and decoders for various formats
+ such as Base64 and Hexadecimal.
+ - http://commons.apache.org/codec/
+
+* log4j 1.2.15 (Apache license 2.0)
+ - With log4j it is possible to enable logging at
+ runtime without modifying the application binary.
+ The log4j package is designed so that these statements
+ can remain in shipped code without incurring a
+ heavy performance cost.
+ - https://logging.apache.org/
+
+* commons-cli 1.1 (Apache license 2.0)
+ - Commons CLI provides a simple API for presenting,
+ processing and validating a command line interface.
+ - http://commons.apache.org/cli/
+
+* commons-logging 1.1.1 (Apache license 2.0)
+ - Commons Logging is a thin adapter allowing
+ configurable bridging to other, well known logging systems.
+ - http://commons.apache.org/logging/
+
+* xstream 1.3.1 (BSD-style license)
+ - XStream is a simple library to serialize objects to XML and back again.
+ - http://xstream.codehaus.org/
+
+* xpp3 1.1.4c (Indiana University Extreme! Lab Software License)
+ - A stable XmlPull parsing engine that is based on ideas from XPP
+ and in particular XPP2 but completely revised and
+ rewritten to take the best advantage of latest JIT JVMs
+ such as Hotspot in JDK 1.4+.
+ - http://www.extreme.indiana.edu/xgws/xsoap/xpp/
+
+* c3p0 0.9.0 (GNU Library or Lesser General Public License)
+ - c3p0 is an easy-to-use library for augmenting traditional
+ (DriverManager-based) JDBC drivers with JNDI-bindable DataSources,
+ including DataSources that implement Connection and Statement
+ Pooling, as described by the jdbc3 spec and jdbc2 std extension.
+ - http://sourceforge.net/projects/c3p0/
+
+* mysql-connector-java 3.0.10 (GPLv2)
+ - MySQL Connector/J is the official JDBC driver for MySQL.
+ - http://dev.mysql.com/downloads/connector/j/
+
+* jgrapht 0.7.3 (GNU Lesser General Public License)
+ - JGraphT is a free Java graph library that provides
+ mathematical graph-theory objects and algorithms.
+ - http://www.jgrapht.org/
+
+---
+
+* GNU licenses
+ - https://www.gnu.org/licenses/
+
+* Xstream license
+ - http://xstream.codehaus.org/license.html
+
+* Apache license 2.0
+ - https://www.apache.org/licenses/LICENSE-2.0.html
+
+* xpp3 license
+ - http://www.extreme.indiana.edu/xgws/xsoap/PullParser/LICENSE.txt
+
+---
+
+1. To compile just install maven and run mvn compile.
+2. To run tests run mvn test.
+3. To package run mvn package.
+4. To clean run mvn clean.
+
+---
+
72 commons/pom.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <groupId>yahoo.flowetl</groupId>
+ <artifactId>super</artifactId>
+ <version>1.0</version>
+ </parent>
+
+ <description>This is the commons module for components and or pipes that can be easily shared among different flows.</description>
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>commons</artifactId>
+ <name>Commons</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>5.11</version>
+ <scope>test</scope>
+ <classifier>jdk15</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial</groupId>
+ <artifactId>sqlite-jdbc</artifactId>
+ <version>3.7.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Not test dependencies -->
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>services</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
103 commons/src/main/java/com/yahoo/flowetl/commons/db/Field.java
@@ -0,0 +1,103 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ *******************************************************/
+package com.yahoo.flowetl.commons.db;
+
+/**
+ * This class represents a field inside a database and its potential rename.
+ *
+ * @author Joshua Harlow
+ */
+public class Field
+{
+
+ /** The field. */
+ private String field;
+
+ /** The rename. */
+ private String rename;
+
+ /**
+ * Instantiates a new field with no rename.
+ *
+ * @param dbField
+ */
+ public Field(String dbField) {
+ this(dbField, null);
+ }
+
+ /**
+ * Instantiates a new field with a rename.
+ *
+ * @param dbField
+ * @param fieldRename
+ */
+ public Field(String dbField, String fieldRename) {
+ super();
+ this.field = dbField;
+ this.rename = fieldRename;
+ }
+
+ /**
+ * Gets the field.
+ *
+ * @return the field
+ */
+ public String getField() {
+ return field;
+ }
+
+ /**
+ * Gets the rename.
+ *
+ * @return the rename
+ */
+ public String getRename() {
+ return rename;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((field == null) ? 0 : field.hashCode());
+ result = prime * result + ((rename == null) ? 0 : rename.hashCode());
+ return result;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Field other = (Field) obj;
+ if (field == null) {
+ if (other.field != null)
+ return false;
+ }
+ else if (!field.equals(other.field))
+ return false;
+ if (rename == null) {
+ if (other.rename != null)
+ return false;
+ }
+ else if (!rename.equals(other.rename))
+ return false;
+ return true;
+ }
+}
369 commons/src/main/java/com/yahoo/flowetl/commons/db/JoinPipe.java
@@ -0,0 +1,369 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ *******************************************************/
+package com.yahoo.flowetl.commons.db;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.yahoo.flowetl.core.CoreException;
+import com.yahoo.flowetl.core.InputValidator;
+import com.yahoo.flowetl.core.Logger.Level;
+import com.yahoo.flowetl.core.pipe.AbstractPipe;
+import com.yahoo.flowetl.core.pipe.PipeResult;
+import com.yahoo.flowetl.core.pipe.result.BackedPipeResult;
+import com.yahoo.flowetl.core.services.ServiceRegistry;
+import com.yahoo.flowetl.core.validator.MapInputValidator;
+import com.yahoo.flowetl.services.DatabaseService;
+
+/**
+ * This class represents a simple mysql based join that takes in a dsn for which
+ * database to perform the join on (or which server). Then it takes in the join
+ * object which will define how to form the join (its nothing complicated). As
+ * output the result set of that join will be provided, or an exception will be
+ * thrown if that result set can not be created.
+ *
+ * @author Joshua Harlow
+ */
+public class JoinPipe extends AbstractPipe
+{
+ // innies required
+ public static final String IN_DSN = makeParamName(JoinPipe.class, "dsn", true);
+ public static final String IN_JOIN = makeParamName(JoinPipe.class, "join", true);
+ private static final Map<String, Class<?>> REQUIRED_PARAMS = new TreeMap<String, Class<?>>();
+ static {
+ REQUIRED_PARAMS.put(IN_JOIN, Join.class);
+ REQUIRED_PARAMS.put(IN_DSN, String.class);
+ }
+
+ // outties
+ public static final String OUT_RESULT_SET = makeParamName(JoinPipe.class, "resultset", false);
+
+ // the output result class
+ // that allows u to extract the result set without doing anything special
+ public static class Result extends BackedPipeResult
+ {
+ public Result() {
+ super();
+ }
+
+ public ResultSet getResultSet() {
+ return getParam(OUT_RESULT_SET);
+ }
+ }
+
+ /**
+ * This class represents our simplified join.
+ */
+ public static class Join
+ {
+
+ // the join type
+ public static enum Type
+ {
+ LEFT, RIGHT
+ }
+
+ /** The fields that we will be selecting on (if empty * will be used) */
+ private final Set<Field> fields;
+
+ /** The src database name. */
+ private final String dbName;
+
+ /** The src table name. */
+ private final String tableName;
+
+ /** The fields and databases/tables we will join with. */
+ private final Map<String, Join> joinWith;
+
+ /** The join type. */
+ private final Type joinType;
+
+ /**
+ * Instantiates a new join.
+ *
+ * @param dbName
+ * @param tableName
+ */
+ public Join(String dbName, String tableName) {
+ this(dbName, tableName, null);
+ }
+
+ /**
+ * Instantiates a new join.
+ *
+ * @param dbName
+ * @param tableName
+ * @param joinType
+ */
+ public Join(String dbName, String tableName, Type joinType) {
+ super();
+ this.dbName = dbName;
+ this.fields = new HashSet<Field>();
+ this.joinWith = new HashMap<String, Join>();
+ this.tableName = tableName;
+ this.joinType = joinType;
+ }
+
+ /**
+ * Gets the database name.
+ *
+ * @return the current src database name
+ */
+ public String getDbName() {
+ return dbName;
+ }
+
+ /**
+ * Adds a field that we will be selecting on.
+ *
+ * @param selectField
+ */
+ public void addField(Field selectField) {
+ fields.add(selectField);
+ }
+
+ /**
+ * Adds the given field and db/table that we will join with
+ *
+ * @param onField
+ * @param who
+ */
+ public void addJoin(String onField, Join who) {
+ joinWith.put(onField, who);
+ }
+
+ /**
+ * Gets the fields.
+ */
+ public Set<Field> getFields() {
+ return fields;
+ }
+
+ /**
+ * Gets who we will join with.
+ */
+ public Map<String, Join> getJoinWith() {
+ return joinWith;
+ }
+
+ /**
+ * Gets the join type.
+ */
+ public Type getJoinType() {
+ return joinType;
+ }
+
+ /**
+ * Gets the src table name.
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ }
+
+ /** The db service for looking up database connections. */
+ private final DatabaseService dbService;
+
+ /** The validator that will check the inputs. */
+ private final InputValidator validator;
+
+ /**
+ * Instantiates a new join pipe.
+ *
+ * @param name
+ * @param services
+ */
+ public JoinPipe(String name, ServiceRegistry services) {
+ super(name, services);
+ this.dbService = services.getService(DatabaseService.class);
+ if (dbService == null) {
+ throw new IllegalArgumentException(JoinPipe.class + " can not operate without a database service");
+ }
+ validator = new MapInputValidator(REQUIRED_PARAMS, null);
+ }
+
+ /**
+ * Makes a db field string by combining the db name with the table name with
+ * the fieldname.
+ *
+ * @param dbName
+ * @param tableName
+ * @param fieldName
+ *
+ * @return the string
+ */
+ public static String makeDbField(String dbName, String tableName, String fieldName) {
+ return StringUtils.join(new Object[] { dbName, tableName, fieldName }, ".");
+ }
+
+ /**
+ * Makes the sql that represents the given join.
+ *
+ * @param start
+ *
+ * @throws IllegalArgumentException
+ * if the input join is not valid
+ *
+ * @return the string
+ */
+ private String makeSql(Join start) {
+ if (StringUtils.isEmpty(start.getDbName())) {
+ throw new IllegalArgumentException("Attempt to make a join without any start database to join on");
+ }
+ if (StringUtils.isEmpty(start.getTableName())) {
+ throw new IllegalArgumentException("Attempt to make a join without any start database table name to join on");
+ }
+ List<String> tmpList = new LinkedList<String>();
+ StringBuilder tmpBuilder = new StringBuilder();
+ for (Field f : start.getFields()) {
+ if (f == null) {
+ continue;
+ }
+ String name = f.getField();
+ if (StringUtils.isEmpty(name)) {
+ continue;
+ }
+ tmpBuilder.append(name);
+ if (StringUtils.isEmpty(f.getRename()) == false) {
+ tmpBuilder.append(" AS ");
+ tmpBuilder.append(f.getRename());
+ }
+ tmpList.add(tmpBuilder.toString());
+ tmpBuilder.setLength(0);
+ }
+ StringBuilder sql = new StringBuilder();
+ sql.append("SELECT ");
+ if (tmpList.isEmpty()) {
+ sql.append(" * ");
+ }
+ else {
+ sql.append(StringUtils.join(tmpList, ", "));
+ }
+ sql.append(" FROM ");
+ sql.append(start.getDbName());
+ sql.append(" ");
+ if (start.getJoinType() == null) {
+ sql.append(Join.Type.RIGHT);
+ }
+ else {
+ sql.append(start.getJoinType());
+ }
+ sql.append(" JOIN ");
+ Set<String> dbNames = new HashSet<String>();
+ for (Entry<String, Join> joinMe : start.getJoinWith().entrySet()) {
+ String thereDbName = joinMe.getValue().getDbName();
+ if (StringUtils.isEmpty(thereDbName)) {
+ continue;
+ }
+ dbNames.add(thereDbName);
+ }
+ if (dbNames.isEmpty()) {
+ // not really a join...
+ throw new IllegalArgumentException("Attempt to make a join without any db names to join on");
+ }
+ sql.append(" (");
+ sql.append(StringUtils.join(dbNames, ", "));
+ sql.append(") ");
+ tmpList.clear();
+ String myName = start.getDbName();
+ String myTable = start.getTableName();
+ tmpBuilder.setLength(0);
+ for (Entry<String, Join> joinMe : start.getJoinWith().entrySet()) {
+ String sharedField = joinMe.getKey();
+ String thereDbName = joinMe.getValue().getDbName();
+ String thereTableName = joinMe.getValue().getTableName();
+ if (StringUtils.isEmpty(thereTableName) || StringUtils.isEmpty(thereDbName) || StringUtils.isEmpty(sharedField)) {
+ continue;
+ }
+ tmpBuilder.append(makeDbField(myName, myTable, sharedField));
+ tmpBuilder.append("=");
+ tmpBuilder.append(makeDbField(thereDbName, thereTableName, sharedField));
+ tmpList.add(tmpBuilder.toString());
+ tmpBuilder.setLength(0);
+ }
+ if (tmpList.isEmpty()) {
+ // not really a join...
+ throw new IllegalArgumentException("Attempt to make a join without any fields to join on");
+ }
+ sql.append(" ON (");
+ sql.append(StringUtils.join(tmpList, " AND "));
+ sql.append(")");
+ return sql.toString();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.yahoo.flowetl.core.pipe.AbstractPipe#maxOutputs()
+ */
+ @Override
+ public int maxOutputs() {
+ // we are outputting a iterator
+ // which should only be used by 1 output
+ // so enforce that...
+ return 1;
+ }
+
+ /**
+ * Executes the join string with the given dsn.
+ *
+ * @param dsn
+ * @param sql
+ *
+ * @return the result set
+ */
+ protected ResultSet executeJoin(String dsn, String sql) {
+ Connection dbConnector = dbService.getConnection(dsn);
+ PreparedStatement call = null;
+ try {
+ call = dbConnector.prepareStatement(sql, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to prepare statement for query " + sql, e);
+ }
+ try {
+ return call.executeQuery();
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to execute prepared statement " + call, e);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.yahoo.flowetl.core.pipe.AbstractPipe#makeOutput(java.util.List)
+ */
+ @Override
+ protected PipeResult makeOutput(List<PipeResult> collectedInputs) {
+ BackedPipeResult merged = new BackedPipeResult(collectedInputs);
+ validator.checkInput(merged);
+ String dsn = merged.getParam(IN_DSN);
+ Join start = merged.getParam(IN_JOIN);
+ Result out = new Result();
+ String sql = makeSql(start);
+ if (getLogger().isEnabled(Level.INFO)) {
+ getLogger().log(Level.INFO, "Running query " + sql + " on dsn " + dsn);
+ }
+ ResultSet rs = executeJoin(dsn, sql);
+ out.setParam(OUT_RESULT_SET, rs);
+ return out;
+ }
+
+}
159 commons/src/main/java/com/yahoo/flowetl/commons/db/SelectPipe.java
@@ -0,0 +1,159 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ *******************************************************/
+package com.yahoo.flowetl.commons.db;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import com.yahoo.flowetl.core.CoreException;
+import com.yahoo.flowetl.core.InputValidator;
+import com.yahoo.flowetl.core.Logger.Level;
+import com.yahoo.flowetl.core.pipe.AbstractPipe;
+import com.yahoo.flowetl.core.pipe.PipeResult;
+import com.yahoo.flowetl.core.pipe.result.BackedPipeResult;
+import com.yahoo.flowetl.core.services.ServiceRegistry;
+import com.yahoo.flowetl.core.validator.MapInputValidator;
+import com.yahoo.flowetl.services.DatabaseService;
+
+/**
+ * This class represents a simple select like pipe for database interactions. It
+ * takes a database dsn and a query with an optional query params list which
+ * will be bound to the prepared statement that is that query (if it has
+ * anything to bind). The result will be a result set object.
+ *
+ * @author Joshua Harlow
+ */
+public class SelectPipe extends AbstractPipe
+{
+ // innies required
+ public static final String IN_DSN = makeParamName(SelectPipe.class, "dsn", true);
+ public static final String IN_SELECT = makeParamName(SelectPipe.class, "query", true);
+ private static final Map<String, Class<?>> REQUIRED_PARAMS = new TreeMap<String, Class<?>>();
+ static {
+ REQUIRED_PARAMS.put(IN_SELECT, String.class);
+ REQUIRED_PARAMS.put(IN_DSN, String.class);
+ }
+
+ // innies optional
+ public static final String IN_SELECT_PARAMS = makeParamName(SelectPipe.class, "queryparams", true);
+ private static final Map<String, Class<?>> OPTIONAL_PARAMS = new TreeMap<String, Class<?>>();
+ static {
+ OPTIONAL_PARAMS.put(IN_SELECT_PARAMS, List.class);
+ }
+
+ // outties
+ public static final String OUT_RESULT_SET = AbstractPipe.makeParamName(SelectPipe.class, "resultset", false);
+
+ // the output result class
+ // that allows u to extract the result set without doing anything special
+ public static class Result extends BackedPipeResult
+ {
+ public Result() {
+ super();
+ }
+
+ public ResultSet getResultSet() {
+ return getParam(OUT_RESULT_SET);
+ }
+ }
+
+ /** The input validation class. */
+ private final InputValidator validator;
+
+ /** The db service. */
+ private final DatabaseService dbService;
+
+ /**
+ * Instantiates a new select pipe.
+ *
+ * @param name
+ * @param services
+ */
+ public SelectPipe(String name, ServiceRegistry services) {
+ super(name, services);
+ validator = new MapInputValidator(REQUIRED_PARAMS, OPTIONAL_PARAMS);
+ dbService = getServiceRegistry().getService(DatabaseService.class);
+ if (dbService == null) {
+ throw new IllegalArgumentException(SelectPipe.class + " requires a database service to operate");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.yahoo.flowetl.core.pipe.AbstractPipe#maxOutputs()
+ */
+ @Override
+ public int maxOutputs() {
+ // we are outputting a iterator
+ // which should only be used by 1 output
+ // so enforce that...
+ return 1;
+ }
+
+ /**
+ * Makes the output for the given input by forming the sql query, binding
+ * the params and then returning the result set.
+ *
+ * @param input
+ * the input
+ *
+ * @return the query result set
+ */
+ private ResultSet makeOutput(PipeResult input) {
+ String dsn = input.getParam(IN_DSN);
+ String query = input.getParam(IN_SELECT);
+ Connection con = dbService.getConnection(dsn);
+ PreparedStatement m = null;
+ if (getLogger().isEnabled(Level.INFO)) {
+ getLogger().log(Level.INFO, "Running query " + query + " : " + dsn);
+ }
+ try {
+ // we won't notice changes by others
+ // and read only
+ m = con.prepareStatement(query, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to prepare statement for query " + query, e);
+ }
+ List<?> pParams = input.getParam(IN_SELECT_PARAMS);
+ if (pParams != null) {
+ for (int i = 0; i < pParams.size(); i++) {
+ try {
+ m.setObject((i + 1), pParams.get(i));
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to set param " + (i + 1) + " on query " + query + " to " + pParams.get(i), e);
+ }
+ }
+ }
+ try {
+ return m.executeQuery();
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to execute prepared statement " + m, e);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.yahoo.flowetl.core.pipe.AbstractPipe#makeOutput(java.util.List)
+ */
+ @Override
+ protected PipeResult makeOutput(List<PipeResult> inputs) {
+ BackedPipeResult merged = new BackedPipeResult(inputs);
+ validator.checkInput(merged);
+ Result res = new Result();
+ res.setParam(OUT_RESULT_SET, makeOutput(merged));
+ return res;
+ }
+}
34 commons/src/main/java/com/yahoo/flowetl/commons/db/actions/Action.java
@@ -0,0 +1,34 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.db.actions;
+
+import java.sql.Connection;
+
+/**
+ * An interface that represents some result of an action on a database
+ * connection.
+ *
+ * @author Joshua Harlow
+ */
+public interface Action<T>
+{
+ /**
+ * Applies some action on a database.
+ *
+ * @param db
+ * the db to apply the action to
+ *
+ * @return the result of that action
+ */
+ public abstract T applyAction(Connection db);
+}
170 commons/src/main/java/com/yahoo/flowetl/commons/db/actions/ActionSequencer.java
@@ -0,0 +1,170 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.db.actions;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.yahoo.flowetl.core.CoreException;
+import com.yahoo.flowetl.core.Logger;
+import com.yahoo.flowetl.core.Logger.Level;
+
+/**
+ * This class processes a set of actions with the capability to rollback those
+ * actions (if those actions can themselves be rolled back as a group).
+ *
+ * @author Joshua Harlow
+ */
+public abstract class ActionSequencer
+{
+ private static final Logger logger = new Logger(ActionSequencer.class);
+
+ /** Should we attempt to do rollback on errors */
+ private final boolean enableRollback;
+
+ /**
+ * Instantiates a new action sequencer.
+ *
+ * @param enableRollback
+ */
+ public ActionSequencer(boolean enableRollback) {
+ this.enableRollback = enableRollback;
+ }
+
+ /**
+ * Instantiates a new action sequencer without rollback support.
+ */
+ public ActionSequencer() {
+ this(false);
+ }
+
+ /**
+ * This can be derived from to determine when a given action is applied and
+ * with what result and if it was committed (or will the commit wait till
+ * the end of all actions).
+ *
+ * @param who
+ * @param whoResult
+ * @param wasCommited
+ */
+ protected void onApplied(Action<?> who, Object whoResult, boolean wasCommited) {
+ // does nothing
+ }
+
+ /**
+ * Executes the given actions.
+ *
+ * @param actions
+ *
+ * @return the list of output from each action
+ */
+ public List<Object> executeActions(List<Action<?>> actions) {
+ if (actions == null || actions.isEmpty()) {
+ return new LinkedList<Object>();
+ }
+ final Connection con = getConnection();
+ boolean oldAuto = true;
+ try {
+ oldAuto = con.getAutoCommit();
+ }
+ catch (SQLException e) {
+ throw new CoreException("Can not read auto commit value for rollback support", e);
+ }
+ // save the state?
+ if (enableRollback == true && oldAuto == true) {
+ try {
+ // gotta turn that off
+ con.setAutoCommit(false);
+ }
+ catch (SQLException e) {
+ throw new CoreException("Can not turn auto commit off for rollback support", e);
+ }
+ }
+ // apply the actions
+ final int am = actions.size();
+ List<Object> out = new ArrayList<Object>(am);
+ int amExec = 0;
+ try {
+ logger.log(Level.INFO, "Executing %s actions", am);
+ for (int i = 0; i < am; i++) {
+ Action<?> a = actions.get(i);
+ logger.log(Level.INFO, "Executing %s : %s action", (i + 1), a);
+ Object res = a.applyAction(con);
+ if (enableRollback) {
+ onApplied(a, res, false);
+ }
+ else {
+ onApplied(a, res, true);
+ }
+ out.add(res);
+ amExec++;
+ }
+ // commit all at the end of apply
+ if (enableRollback == true) {
+ con.commit();
+ }
+ }
+ catch (SQLException e) {
+ if (enableRollback) {
+ try {
+ con.rollback();
+ }
+ catch (SQLException e1) {
+ logger.log(Level.WARN, "Unable to perform rollback", e1);
+ }
+ }
+ // rethrow
+ throw new CoreException("Unable to commit due to sql error", e);
+ }
+ catch (RuntimeException e) {
+ if (enableRollback) {
+ try {
+ con.rollback();
+ }
+ catch (SQLException e1) {
+ logger.log(Level.WARN, "Unable to perform rollback", e1);
+ }
+ }
+ // rethrow
+ throw e;
+ }
+ finally {
+ if (oldAuto == true && enableRollback == true) {
+ try {
+ con.setAutoCommit(oldAuto);
+ }
+ catch (SQLException e) {
+ logger.log(Level.WARN, "Unable to restore auto commit value to " + oldAuto, e);
+ }
+ }
+ try {
+ con.close();
+ }
+ catch (SQLException e) {
+ logger.log(Level.WARN, "Unable to close connection", e);
+ }
+ }
+ return out;
+ }
+
+ /**
+ * Gets the db connection for the actions to use.
+ *
+ * @return the connection
+ */
+ protected abstract Connection getConnection();
+
+}
117 commons/src/main/java/com/yahoo/flowetl/commons/db/actions/UpdateAction.java
@@ -0,0 +1,117 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.db.actions;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.yahoo.flowetl.core.CoreException;
+import com.yahoo.flowetl.core.Logger;
+import com.yahoo.flowetl.core.Logger.Level;
+
+/**
+ * This class represents a update action on a database which maps to a sql
+ * update. It returns the number of records that that update affected.
+ *
+ * @author Joshua Harlow
+ */
+public abstract class UpdateAction implements Action<Integer>
+{
+ private static final Logger logger = new Logger(UpdateAction.class);
+
+ /**
+ * Instantiates a new update action.
+ */
+ public UpdateAction() {
+ super();
+ }
+
+ /**
+ * This should return the sql for the given update
+ */
+ protected abstract String getSql();
+
+ /**
+ * Gets the bindings for that sql if any.
+ */
+ protected abstract List<Object> getBindings();
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * com.yahoo.flowetl.commons.db.actions.Action#applyAction(java.sql.Connection
+ * )
+ */
+ @Override
+ public Integer applyAction(Connection db) {
+ PreparedStatement st = null;
+ try {
+ String sql = getSql();
+ try {
+ st = db.prepareStatement(sql);
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to prepare update action for query " + sql, e);
+ }
+ List<Object> binds = getBindings();
+ if (binds != null && binds.isEmpty() == false) {
+ for (int i = 0; i < binds.size(); i++) {
+ Object o = binds.get(i);
+ try {
+ st.setObject((i + 1), o);
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to bind param " + (i + 1) + " for sql " + sql + " with param " + o, e);
+ }
+ }
+ }
+ try {
+ logger.log(Level.DEBUG, "Executing %s with bound params %s", sql, binds);
+ int res = st.executeUpdate();
+ logger.log(Level.DEBUG, "Executing %s affected %s rows", sql, res);
+ return res;
+ }
+ catch (SQLException e) {
+ throw new CoreException("Unable to execute update for sql " + sql + " with params " + StringUtils.join(binds, ","), e);
+ }
+ }
+ finally {
+ if (st != null) {
+ try {
+ st.close();
+ }
+ catch (SQLException e) {
+ logger.log(Level.WARN, "Error closing update action prepared statement", e);
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(super.toString() + " [sql=" + getSql() + "][bindings=" + getBindings() + "]");
+ return builder.toString();
+ }
+
+}
138 commons/src/main/java/com/yahoo/flowetl/commons/runner/BaseRunner.java
@@ -0,0 +1,138 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.runner;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang.StringUtils;
+
+import com.yahoo.flowetl.core.PipeRunner;
+
+/**
+ * This class helps create a main program that will activate a given set of
+ * pipes. It helps by giving you overload points that can add on different
+ * config that is needed while still allowing you to specify what options are
+ * valid for your programs consumption.
+ *
+ * @author Joshua Harlow
+ */
+public abstract class BaseRunner implements Runner
+{
+ // default config names
+ protected static final String CONFIG_OPT = "config";
+ protected static final String HELP_OPT = "help";
+
+ // program name
+ private static final String PROG_NAME = "FlowEtl";
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * com.yahoo.flowetl.commons.runner.Runner#runProgram(java.lang.String[])
+ */
+ public final void runProgram(String[] args) throws Exception {
+ CommandLineParser parser = new PosixParser();
+ Options opts = getOptions();
+ CommandLine cmd = parser.parse(opts, args);
+ if (helpNeeded(cmd)) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(getProgramName(), opts);
+ return;
+ }
+ runProgram(cmd);
+ }
+
+ /**
+ * Gets the program name which we are running as. Can be overriden to change
+ * as desired...
+ */
+ protected String getProgramName() {
+ return PROG_NAME;
+ }
+
+ /**
+ * Should return whether help is needed or whether it is not needed by
+ * examining the command line options that have been parsed and determining
+ * if they are sufficient.
+ *
+ * @param cmd
+ *
+ * @return if help is needed or not
+ */
+ protected boolean helpNeeded(CommandLine cmd) {
+ if (cmd.hasOption(HELP_OPT)) {
+ return true;
+ }
+ String cfgFile = cmd.getOptionValue(CONFIG_OPT);
+ if (StringUtils.isEmpty(cfgFile)) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * The override point where you need to create the pipe runner that will
+ * perform your actions, aka the pipes that this program should run. At this
+ * point no help will be needed but you still have to analyze the command
+ * line parsed options to determine what is needed to construct your pipes.
+ *
+ * @param cmd
+ * the cmd line options
+ *
+ * @throws Exception
+ * if forming a pipe runner fails
+ */
+ protected abstract PipeRunner formRunner(CommandLine cmd) throws Exception;
+
+ /**
+ * Runs the program by taking the given options and forming a runner and
+ * adding on a simple command line listener to that pipe runner and then
+ * running the pipe which was created in the currently active thread.
+ *
+ * @param cmd
+ * the cmd line options (post help validation)
+ *
+ * @throws Exception
+ * the exception
+ */
+ protected void runProgram(CommandLine cmd) throws Exception {
+ // setup some default job
+ PipeRunner toRun = formRunner(cmd);
+ // make a cmd line output listener
+ toRun.addFlowListener(new CommandLineListener());
+ // do it...
+ toRun.run();
+ }
+
+ /**
+ * Gets the command line options that this class will be using.
+ *
+ * @return the options
+ */
+ @SuppressWarnings("static-access")
+ protected Options getOptions() {
+ Options cliOpts = new Options();
+ Option cfgFile = OptionBuilder.withArgName("file").hasArg(true).withDescription("the configuration file").create(CONFIG_OPT);
+ cliOpts.addOption(cfgFile);
+ Option helpOpt = OptionBuilder.hasArg(false).withDescription("help message").create(HELP_OPT);
+ cliOpts.addOption(helpOpt);
+ return cliOpts;
+ }
+
+}
236 commons/src/main/java/com/yahoo/flowetl/commons/runner/CommandLineListener.java
@@ -0,0 +1,236 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.runner;
+
+import java.io.PrintStream;
+import java.text.NumberFormat;
+import java.util.List;
+
+import com.yahoo.flowetl.core.listener.FlowListener;
+import com.yahoo.flowetl.core.pipe.Pipe;
+import com.yahoo.flowetl.core.pipe.PipeResult;
+
+/**
+ * This class is a simple flow listener which will show status on a command line
+ * terminal (or the like). It gets notified of pipe running and outputs this
+ * information while also having a separate thread that shows the
+ * status/progress of the current pipe.
+ *
+ * @author Joshua Harlow
+ */
+public class CommandLineListener implements FlowListener
+{
+ // various static objects/constants used
+ private static final NumberFormat PER_FORMAT = NumberFormat.getPercentInstance();
+ private static final long MEGABYTE = 1024L * 1024L;
+ // how often we show a dot, everyone 1/2 second
+ private static final long WAIT_MS = 500;
+ // how often we see if the progress of the current pipe has changed
+ // ie every 25 dots
+ private static final int PER_SHOW = 25;
+ // a little dot will come out on the command line
+ private static final String STATUS_INDICATOR = ".";
+
+ // these
+ // need to be accessed in thread safe way...
+ private final Object statusLock = new Object();
+ private boolean showStatus = false;
+ private int numStatusOuts = 0;
+ private float lastPerDone = 0;
+ private Pipe activePipe = null;
+ //
+
+ // the output stream, typically stdout but could change
+ private final PrintStream out;
+
+ // the thread that is showing the pipe actions/progress...
+ private final Thread showActions;
+
+ // the total number of pipes that will run
+ private int totalPipes;
+
+ // the currently active pipe number (less than totalpipes)
+ private int currPipe;
+
+ // this is the thread that will show the current pipes status
+ private final class StatusIndicatorRunnable implements Runnable
+ {
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Runnable#run()
+ */
+ @Override
+ public void run() {
+ while (true) {
+ synchronized (statusLock) {
+ if (showStatus) {
+ out.print(STATUS_INDICATOR);
+ numStatusOuts++;
+ // time to show how far along the current pipe is??
+ if ((numStatusOuts % PER_SHOW) == 0 && activePipe != null) {
+ float curPercentDone = activePipe.percentDone();
+ if (curPercentDone != lastPerDone) {
+ out.print(formatDonePercent(curPercentDone));
+ lastPerDone = curPercentDone;
+ }
+ }
+ }
+ }
+ try {
+ Thread.sleep(WAIT_MS);
+ }
+ catch (InterruptedException e) {
+ }
+ }
+ }
+ }
+
+ /**
+ * Instantiates a new command line listener using system.out as the
+ * outputstream.
+ */
+ public CommandLineListener() {
+ this(System.out);
+ }
+
+ /**
+ * Instantiates a new command line listener.
+ *
+ * @param out
+ * the output stream to write to
+ */
+ public CommandLineListener(PrintStream out) {
+ this.out = out;
+ showActions = new Thread(new StatusIndicatorRunnable());
+ showActions.setDaemon(true);
+ showActions.start();
+ }
+
+ /**
+ * Formats a float done percent to a more meaningful string.
+ *
+ * @param perDone
+ *
+ * @return the string
+ */
+ private static String formatDonePercent(float perDone) {
+ return "%D{" + PER_FORMAT.format(perDone) + "}";
+ }
+
+ /**
+ * Gets the heap info of the current JVM as a string
+ *
+ * @param showMax
+ * if we should show the max heap size (not to useful after
+ * showing it once)
+ *
+ * @return the heap info string
+ */
+ private static String getHeapInfo(boolean showMax) {
+ long heapSize = Runtime.getRuntime().totalMemory();
+ long heapFreeSize = Runtime.getRuntime().freeMemory();
+ StringBuilder out = new StringBuilder();
+ out.append("size = " + (heapSize / MEGABYTE) + "Mb, ");
+ if (showMax) {
+ long heapMaxSize = Runtime.getRuntime().maxMemory();
+ out.append("max size = " + (heapMaxSize / MEGABYTE) + "Mb, ");
+ }
+ out.append("free size = " + (heapFreeSize / MEGABYTE) + "Mb");
+ return "%H{" + out.toString() + "}";
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * com.yahoo.flowetl.core.listener.FlowListener#onStartGenerate(com.yahoo
+ * .flowetl.core.pipe.Pipe)
+ */
+ @Override
+ public void onStartGenerate(Pipe aboutToRunPipe) {
+ out.println("Running pipe #" + (currPipe + 1) + " {" + aboutToRunPipe + "}");
+ out.println("Heap info is " + getHeapInfo(false) + "");
+ synchronized (statusLock) {
+ activePipe = aboutToRunPipe;
+ numStatusOuts = 0;
+ showStatus = true;
+ lastPerDone = 0;
+ // show starting 0.0%
+ out.print(formatDonePercent(0.0f));
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.yahoo.flowetl.core.listener.FlowListener#onStart(java.util.List)
+ */
+ @Override
+ public void onStart(List<Pipe> ordering) {
+ totalPipes = ordering.size();
+ currPipe = 0;
+ out.println("About to start running " + totalPipes + " pipes");
+ for (int i = 0; i < ordering.size(); i++) {
+ out.println(" (" + (i + 1) + ") " + ordering.get(i));
+ }
+ synchronized (statusLock) {
+ numStatusOuts = 0;
+ lastPerDone = 0;
+ showStatus = false;
+ activePipe = null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * com.yahoo.flowetl.core.listener.FlowListener#onFinishGenerate(com.yahoo
+ * .flowetl.core.pipe.Pipe, com.yahoo.flowetl.core.pipe.PipeResult, long)
+ */
+ @Override
+ public void onFinishGenerate(Pipe ranPipe, PipeResult ranResult, long timeTakenMs) {
+ synchronized (statusLock) {
+ showStatus = false;
+ activePipe = null;
+ numStatusOuts = 0;
+ if (lastPerDone < 1) {
+ // show end 100%
+ out.println(formatDonePercent(1));
+ }
+ else {
+ // already showed...
+ out.println();
+ }
+ lastPerDone = 0;
+ }
+ double tSecs = (double) timeTakenMs / 1000.0d;
+ out.println("Finished pipe #" + (currPipe + 1) + " in " + tSecs + " seconds");
+ out.println("With result " + ranResult);
+ currPipe += 1;
+ out.println("Completed - " + formatDonePercent(((float) (currPipe) / (float) totalPipes)) + " of all pipes");
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.yahoo.flowetl.core.listener.FlowListener#onCompletion(long)
+ */
+ @Override
+ public void onCompletion(long timeTakenMs) {
+ double tSecs = (double) timeTakenMs / 1000.0d;
+ out.println("Finished in " + tSecs + " seconds or " + (tSecs / 60) + " minutes");
+ }
+
+}
125 commons/src/main/java/com/yahoo/flowetl/commons/runner/Main.java
@@ -0,0 +1,125 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.runner;
+
+import java.nio.charset.Charset;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.ClassUtils;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang.WordUtils;
+
+import com.yahoo.flowetl.core.util.KlassUtils;
+import com.yahoo.flowetl.core.util.NetUtils;
+
+/**
+ * This class acts as the main entry point which can create a given runner class
+ * and then proxy to it for running whichever actions it decides are necessary.
+ * This is nicer than setting up X main functions and then having to handle all
+ * of those. Instead you just make sure your class to run is a runner class and
+ * it does most of the work for you.
+ *
+ * @author Joshua Harlow
+ */
+public class Main
+{
+ // used for showing that it is...
+ private static final Charset DEF_CHAR_SET = Charset.defaultCharset();
+
+ private Main() {
+ // not creatable...
+ }
+
+ /**
+ * The main method entry point.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length == 0) {
+ System.out.println(Main.class.getSimpleName() + " [runner fully qualified java class name] arguments ...");
+ return;
+ }
+ System.out.println("+Argument info:");
+ StringBuilder argsStr = new StringBuilder();
+ for (int i = 0; i < args.length; i++) {
+ argsStr.append("(" + (i + 1) + ") " + args[i] + " [" + args[i].length() + " chars]");
+ if (i + 1 != args.length) {
+ argsStr.append(" ");
+ }
+ }
+ System.out.println(argsStr);
+ Map<String, Object> sysInfo = getRuntimeInfo();
+ System.out.println("+Runtime info:");
+ for (Entry<String, Object> e : sysInfo.entrySet()) {
+ System.out.println("--- " + e.getKey() + " => " + (e.getValue() == null ? "" : e.getValue()));
+ }
+ String classToRun = args[0];
+ Class<?> testToRun = KlassUtils.getClassForName(classToRun);
+ if (KlassUtils.isAbstract(testToRun) || KlassUtils.isInterface(testToRun)) {
+ System.out.println("+Runner class name that is not abstract or an interface is required!");
+ return;
+ }
+ if (ClassUtils.isAssignable(testToRun, (Runner.class)) == false) {
+ System.out.println("+Runner class name that is a instance/subclass of " + Runner.class.getSimpleName() + " is required!");
+ return;
+ }
+ Class<Runner> rToRun = KlassUtils.getClassForName(classToRun);
+ System.out.println("+Running program specified by runner class " + rToRun);
+ Runner r = KlassUtils.getInstanceOf(rToRun, new Object[] {});
+ String[] nargs = (String[]) ArrayUtils.subarray(args, 1, args.length);
+ System.out.println("+Proxying to object " + r + " with arguments [" + StringUtils.join(nargs, ",") + "]");
+ r.runProgram(nargs);
+ }
+
+ /**
+ * Gets some useful runtime info as a map of names -> info.
+ */
+ private static Map<String, Object> getRuntimeInfo() {
+ Map<String, Object> sysInfo = new TreeMap<String, Object>();
+ StringBuilder jvminfo = new StringBuilder();
+ jvminfo.append("Vendor: ");
+ jvminfo.append(SystemUtils.JAVA_VENDOR);
+ jvminfo.append(", Version: ");
+ jvminfo.append(SystemUtils.JAVA_VERSION + " - " + SystemUtils.JAVA_VM_INFO);
+ jvminfo.append(", OS: ");
+ jvminfo.append(SystemUtils.OS_NAME + " (" + SystemUtils.OS_VERSION + " : " + SystemUtils.OS_ARCH + ")");
+ sysInfo.put(WordUtils.capitalizeFully("jvm"), jvminfo.toString());
+ sysInfo.put(WordUtils.capitalizeFully("default charset encoding"), DEF_CHAR_SET.name());
+ String netAdd = NetUtils.getLocalAddress();
+ if (StringUtils.isEmpty(netAdd)) {
+ netAdd = "????";
+ }
+ String localName = NetUtils.getLocalHostName();
+ if (StringUtils.isEmpty(localName)) {
+ localName = "????";
+ }
+ sysInfo.put(WordUtils.capitalizeFully("network"), localName + " at ip address " + netAdd);
+ String cPath = SystemUtils.JAVA_CLASS_PATH;
+ String linesep = StringEscapeUtils.escapeJava(SystemUtils.LINE_SEPARATOR);
+ sysInfo.put(WordUtils.capitalizeFully("classpath"), cPath);
+ sysInfo.put(WordUtils.capitalizeFully("jvm home"), SystemUtils.JAVA_HOME);
+ sysInfo.put(WordUtils.capitalizeFully("jvm tmpdir"), SystemUtils.JAVA_IO_TMPDIR);
+ sysInfo.put(WordUtils.capitalizeFully("jvm libpath"), SystemUtils.JAVA_LIBRARY_PATH);
+ sysInfo.put(WordUtils.capitalizeFully("line separator"), linesep);
+ sysInfo.put(WordUtils.capitalizeFully("path separator"), StringEscapeUtils.escapeJava(SystemUtils.PATH_SEPARATOR));
+ sysInfo.put(WordUtils.capitalizeFully("user timezone"), SystemUtils.USER_TIMEZONE);
+ sysInfo.put(WordUtils.capitalizeFully("user home"), SystemUtils.USER_HOME);
+ sysInfo.put(WordUtils.capitalizeFully("user language"), SystemUtils.USER_LANGUAGE);
+ sysInfo.put(WordUtils.capitalizeFully("user name"), SystemUtils.USER_NAME);
+ return sysInfo;
+ }
+}
27 commons/src/main/java/com/yahoo/flowetl/commons/runner/Runner.java
@@ -0,0 +1,27 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.runner;
+
+/**
+ * This is what a runner class which the main class we have provided can proxy
+ * to. Most people will choose to extend from base runner which provides more
+ * useful operations than this simple interface.
+ *
+ * @author Joshua Harlow
+ */
+public interface Runner
+{
+ // runs the given program with the given arguments
+ // throws if it can not run...
+ public void runProgram(String[] args) throws Exception;
+}
161 commons/src/main/java/com/yahoo/flowetl/commons/web/HttpPipe.java
@@ -0,0 +1,161 @@
+/*******************************************************
+ * *
+ * Copyright (C) 2011 Yahoo! Inc. All Rights Reserved. *
+ * *
+ * Licensed under the New *
+ * BSD License. See the *
+ * accompanying LICENSE file *
+ * for the specific language *
+ * governing permissions and *
+ * limitations under the *
+ * License. *
+ *******************************************************/
+package com.yahoo.flowetl.commons.web;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.yahoo.flowetl.core.CoreException;
+import com.yahoo.flowetl.core.InputValidator;
+import com.yahoo.flowetl.core.pipe.AbstractPipe;
+import com.yahoo.flowetl.core.pipe.PipeResult;
+import com.yahoo.flowetl.core.pipe.result.BackedPipeResult;
+import com.yahoo.flowetl.core.services.ServiceRegistry;
+import com.yahoo.flowetl.core.validator.MapInputValidator;
+import com.yahoo.flowetl.services.HttpService;
+import com.yahoo.flowetl.services.HttpService.GetHttpParams;
+import com.yahoo.flowetl.services.HttpService.HttpParams;
+import com.yahoo.flowetl.services.HttpService.HttpResult;
+import com.yahoo.flowetl.services.HttpService.PostHttpParams;
+
+/**
+ * This is a pipe which can take some http params as input and give a http
+ * result as output. It can be used by other classes if they find it useable, or
+ * other classes can use the http service directly as they choose.
+ *
+ * @author Joshua Harlow
+ */
+public class HttpPipe extends AbstractPipe
+{
+ // innies required
+ public static final String IN_URI = makeParamName(HttpPipe.class, "uri", true);
+ private static final Map<String, Class<?>> REQUIRED_PARAMS = new TreeMap<String, Class<?>>();
+ static {
+ REQUIRED_PARAMS.put(IN_URI, URI.class);
+ }
+
+ // innies optional
+ public static final String IN_METHOD = makeParamName(HttpPipe.class, "method", true);
+ public static final String IN_TIMEOUT_MS = makeParamName(HttpPipe.class, "in-timeout-ms", true);
+ public static final String IN_POST_DATA = makeParamName(HttpPipe.class, "in-post-data", true);
+ public static final String IN_USER_AGENT = makeParamName(HttpPipe.class, "in-useragent", true);
+ private static final Map<String, Class<?>> OPTIONAL_PARAMS = new TreeMap<String, Class<?>>();
+ static {
+ OPTIONAL_PARAMS.put(IN_METHOD, String.class);
+ OPTIONAL_PARAMS.put(IN_TIMEOUT_MS, Integer.class);
+ OPTIONAL_PARAMS.put(IN_POST_DATA, String.class);
+ OPTIONAL_PARAMS.put(IN_USER_AGENT, String.class);
+ }
+
+ // outties
+ public static final String OUT_STATUS_CODE = makeParamName(HttpPipe.class, "statusCode", false);
+ public static final String OUT_RESPONSE_BODY = makeParamName(HttpPipe.class, "responseBody", false);
+ public static final String OUT_RESPONSE_HEADERS = makeParamName(HttpPipe.class, "responseHeaders", false);
+
+ // the output result class
+ // that allows u to extract the result set without doing anything special
+ public static class Result extends BackedPipeResult
+ {
+ public Result() {
+ super();
+ }
+
+ public Integer getStatusCode() {
+ return getParam(OUT_STATUS_CODE);
+ }
+
+ public String getResponseBody() {
+ return getParam(OUT_RESPONSE_BODY);
+ }
+
+ public Map<String, String> getResponseHeaders() {
+ return getParam(OUT_RESPONSE_HEADERS);
+ }
+ }
+
+ /** The input validation checker. */
+ private final InputValidator inputChecker;
+
+ // required http service
+ private final HttpService httpService;
+
+ /**
+ * Instantiates a new http pipe.