Permalink
Browse files

merging leslie branch

  • Loading branch information...
2 parents 63edaf5 + f4ceb85 commit 33a9c69eb9eab0921f89b4a752b088f283ce57d5 @joshbronson joshbronson committed Dec 20, 2013
View
@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org'
gemspec
View
@@ -26,6 +26,9 @@ def self.configure settings, program
settings.define(:es_id_field, :description => "If this field is present in a record, make an update request, otherwise make a create request", :wukong_hadoop => true)
settings.define(:es_bulk_size, :description => "Number of requests to batch locally before making a request to ElasticSearch", :type => Integer, :wukong_hadoop => true)
settings.define(:es_query, :description => "Query to use when defining input splits for ElasticSearch input", :wukong_hadoop => true)
+ settings.define(:es_transport, :description => "Use a transport client to an existing node instead of spinning up a new node", :default => true, type: :boolean, :wukong_hadoop => true)
+ settings.define(:es_transport_host, :description => "Host of existing node for transport client", default: 'localhost', :wukong_hadoop => true)
+ settings.define(:es_transport_port, :description => "Port of existing node for transport client", default: 9300, type: Integer, :wukong_hadoop => true)
end
# Boot Wonderdog with the given `settings` in the given `dir`.
@@ -105,7 +105,10 @@ def hadoop_jobconf_options
super() + [].tap do |o|
if (reads_from_elasticsearch? || writes_to_elasticsearch?)
- o << java_opt('es.config', settings[:es_config])
+ o << java_opt('es.config', settings[:es_config])
+ o << java_opt('elasticsearch.transport', settings[:es_transport])
+ o << java_opt('elasticsearch.transport.host', settings[:es_transport_host]) if settings[:es_transport] && settings[:es_transport_host]
+ o << java_opt('elasticsearch.transport.port', settings[:es_transport_port]) if settings[:es_transport] && settings[:es_transport_port]
end
if reads_from_elasticsearch?
View
@@ -1,49 +0,0 @@
-Running replay
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-2013-05-03 18:32:24 +0000 2013-03-29 02:15:53,898 520.1ms 255 311 padraig-search-0 browser_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:32:24 +0000 2013-03-29 02:15:54,848 1.4s 600 636 padraig-search-0 flight_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:32:25 +0000 2013-03-29 02:15:54,897 1.5s 546 584 padraig-search-0 flight_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:32:25 +0000 2013-03-29 02:16:00,296 6.9s 7795 7832 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:32:33 +0000 2013-03-29 02:16:00,437 7s 6705 6747 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:32:40 +0000 2013-03-29 02:16:05,408 12s 25336 25374 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:33:05 +0000 2013-03-29 02:16:05,542 12.1s 21673 21711 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:33:27 +0000 2013-03-29 02:16:22,921 1.9s 6625 6685 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:33:34 +0000 2013-03-29 02:16:22,973 2s 6717 6760 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:33:40 +0000 2013-03-29 02:16:26,574 5.6s 21212 21248 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:34:02 +0000 2013-03-29 02:16:26,574 5.5s 21309 21354 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:34:23 +0000 2013-03-29 02:16:30,698 1.8s 6697 6741 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:34:30 +0000 2013-03-29 02:16:30,804 1.9s 6676 6736 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:34:36 +0000 2013-03-29 02:16:34,568 5.6s 21115 21149 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:34:58 +0000 2013-03-29 02:16:34,637 5.7s 21365 21403 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:35:19 +0000 2013-03-29 03:05:53,872 679.6ms 121 157 padraig-search-0 browser_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:19 +0000 2013-03-29 03:05:54,131 938ms 66 102 padraig-search-0 browser_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:19 +0000 2013-03-29 03:05:54,855 1.6s 406 441 padraig-search-0 flight_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:20 +0000 2013-03-29 03:05:55,240 2s 260 303 padraig-search-0 flight_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:20 +0000 2013-03-29 03:05:56,145 2.9s 657 697 padraig-search-0 interaction_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:21 +0000 2013-03-29 03:05:56,288 3s 444 490 padraig-search-0 interaction_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:21 +0000 2013-03-29 03:05:56,318 3.1s 746 811 padraig-search-0 ad_tag_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:22 +0000 2013-03-29 03:05:56,619 3.4s 460 508 padraig-search-0 ad_tag_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:23 +0000 2013-03-29 03:05:56,755 3.5s 1055 1091 padraig-search-0 creative_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:24 +0000 2013-03-29 03:05:57,082 3.8s 639 676 padraig-search-0 creative_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:25 +0000 2013-03-29 03:05:58,424 5.2s 2910 2956 padraig-search-0 placement_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:28 +0000 2013-03-29 03:05:58,499 5.3s 1995 2041 padraig-search-0 placement_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:35:30 +0000 2013-03-29 03:06:13,272 20s 34998 35038 padraig-search-0 site_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-2013-05-03 18:36:05 +0000 2013-03-29 03:06:13,378 20.1s 25515 25552 padraig-search-0 site_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:36:30 +0000 2013-03-29 05:01:05,277 7s 7s 41 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:36:30 +0000 2013-03-29 05:01:05,339 7s 7s 43 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:36:30 +0000 2013-03-29 05:03:23,505 4.6s 4.6s 30 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:36:30 +0000 2013-03-29 05:03:23,610 4.7s 4.7s 30 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:36:30 +0000 2013-03-29 05:03:54,597 6.8s 6.8s 31 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:36:30 +0000 2013-03-29 05:03:54,662 6.9s 6.9s 34 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:36:30 +0000 2013-03-29 05:20:02,276 4.9s 1846 1884 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:36:32 +0000 2013-03-29 05:20:02,311 5s 1845 1889 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:36:34 +0000 2013-03-29 05:25:13,568 2.4s 2.4s 37 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:36:34 +0000 2013-03-29 05:25:13,568 2.4s 2.4s 37 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:36:34 +0000 2013-03-29 05:25:59,835 4.2s 1835 1874 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:36:36 +0000 2013-03-29 05:25:59,865 4.2s 1838 1882 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-All together the slow logs took: 4.232433333333334min
@@ -1,53 +0,0 @@
-Running replay
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-2013-05-03 18:50:41 +0000 2013-03-29 02:15:53,898 520.1ms 216 253 padraig-search-0 browser_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:50:41 +0000 2013-03-29 02:15:54,848 1.4s 616 654 padraig-search-0 flight_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:50:42 +0000 2013-03-29 02:15:54,897 1.5s 509 545 padraig-search-0 flight_count {"query":{"constant_score":{"filter":{"range":{"tb
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-2013-05-03 18:50:42 +0000 2013-03-29 02:16:00,296 6.9s 7777 7811 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:50:50 +0000 2013-03-29 02:16:00,437 7s 6746 6782 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:50:57 +0000 2013-03-29 02:16:05,408 12s 26751 26790 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:51:24 +0000 2013-03-29 02:16:05,542 12.1s 22706 22742 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:51:46 +0000 2013-03-29 02:16:22,921 1.9s 6720 6756 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:51:53 +0000 2013-03-29 02:16:22,973 2s 6677 6735 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:52:00 +0000 2013-03-29 02:16:26,574 5.6s 22334 22369 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:52:22 +0000 2013-03-29 02:16:26,574 5.5s 22702 22737 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:52:45 +0000 2013-03-29 02:16:30,698 1.8s 6677 6713 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:52:52 +0000 2013-03-29 02:16:30,804 1.9s 6634 6673 padraig-search-0 placement_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:52:58 +0000 2013-03-29 02:16:34,568 5.6s 22575 22612 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-2013-05-03 18:53:21 +0000 2013-03-29 02:16:34,637 5.7s 22689 22729 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:53:44 +0000 2013-03-29 03:05:53,872 679.6ms 122 165 padraig-search-0 browser_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:44 +0000 2013-03-29 03:05:54,131 938ms 67 114 padraig-search-0 browser_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:44 +0000 2013-03-29 03:05:54,855 1.6s 404 439 padraig-search-0 flight_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:45 +0000 2013-03-29 03:05:55,240 2s 259 296 padraig-search-0 flight_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:45 +0000 2013-03-29 03:05:56,145 2.9s 651 693 padraig-search-0 interaction_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:46 +0000 2013-03-29 03:05:56,288 3s 446 493 padraig-search-0 interaction_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:46 +0000 2013-03-29 03:05:56,318 3.1s 730 794 padraig-search-0 ad_tag_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-
-current_timestamp original_timestamp es_duration(ms) new_duration(ms) clock_time_duration(ms) node index query_fragment
-2013-05-03 18:53:47 +0000 2013-03-29 03:05:56,619 3.4s 460 500 padraig-search-0 ad_tag_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:48 +0000 2013-03-29 03:05:56,755 3.5s 1036 1079 padraig-search-0 creative_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:49 +0000 2013-03-29 03:05:57,082 3.8s 641 678 padraig-search-0 creative_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:49 +0000 2013-03-29 03:05:58,424 5.2s 2905 2944 padraig-search-0 placement_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:52 +0000 2013-03-29 03:05:58,499 5.3s 2003 2039 padraig-search-0 placement_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:53:54 +0000 2013-03-29 03:06:13,272 20s 39022 39058 padraig-search-0 site_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:54:33 +0000 2013-03-29 03:06:13,378 20.1s 30197 30251 padraig-search-0 site_count {"query":{"match_all":{}},"facets":{"igna":{"stati
-2013-05-03 18:55:04 +0000 2013-03-29 05:01:05,277 7s 7s 36 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:55:04 +0000 2013-03-29 05:01:05,339 7s 7s 30 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:55:04 +0000 2013-03-29 05:03:23,505 4.6s 4.6s 52 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:55:04 +0000 2013-03-29 05:03:23,610 4.7s 4.7s 31 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:55:04 +0000 2013-03-29 05:03:54,597 6.8s 6.8s 34 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:55:04 +0000 2013-03-29 05:03:54,662 6.9s 6.9s 34 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"range":{"tb
-2013-05-03 18:55:04 +0000 2013-03-29 05:20:02,276 4.9s 1841 1876 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:55:06 +0000 2013-03-29 05:20:02,311 5s 1839 1883 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:55:08 +0000 2013-03-29 05:25:13,568 2.4s 2.4s 48 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:55:08 +0000 2013-03-29 05:25:13,568 2.4s 2.4s 37 padraig-search-0 low_precision {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:55:08 +0000 2013-03-29 05:25:59,835 4.2s 1837 1875 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-2013-05-03 18:55:10 +0000 2013-03-29 05:25:59,865 4.2s 1837 1880 padraig-search-0 site_count {"query":{"constant_score":{"filter":{"term":{"fli
-All together the slow logs took: 4.504333333333333min
Oops, something went wrong.

0 comments on commit 33a9c69

Please sign in to comment.