Permalink
Browse files

Travis' script to add warmers to add indices

  • Loading branch information...
1 parent 06e204a commit 5f816d85393d59a3c0eab6968702df5349497108 Missy committed May 7, 2013
Showing with 392 additions and 0 deletions.
  1. +327 −0 test/mapping.yml
  2. +65 −0 test/warmindices.rb
View
@@ -0,0 +1,327 @@
+ad_tag_count:
+ _all:
+ enabled: false
+ _routing:
+ path: ad_tag_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ ad_tag_id:
+ type: integer
+ creative_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ base_feature:
+ index: not_analyzed
+ type: string
+ seconds:
+ type: integer
+flight_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ flight_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ base_feature:
+ index: not_analyzed
+ type: string
+ seconds:
+ type: integer
+ metric_feature:
+ index: not_analyzed
+ type: string
+creative_count:
+ _all:
+ enabled: false
+ _routing:
+ path: creative_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ creative_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ base_feature:
+ index: not_analyzed
+ type: string
+ seconds:
+ type: integer
+item_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ item_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ flight_id:
+ type: integer
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ base_feature:
+ index: not_analyzed
+ type: string
+signal_value_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ signal_value_id:
+ index: not_analyzed
+ type: string
+ flight_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ base_feature:
+ index: not_analyzed
+ type: string
+placement_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ ad_tag_id:
+ type: integer
+ flight_id:
+ type: integer
+ ext_pl_id:
+ index: not_analyzed
+ type: string
+ creative_id:
+ type: integer
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ index: not_analyzed
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ detail:
+ index: not_analyzed
+ type: string
+ pl_composite:
+ index: not_analyzed
+ type: string
+site_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ ad_tag_id:
+ type: integer
+ flight_id:
+ type: integer
+ ext_site_id:
+ index: not_analyzed
+ type: string
+ creative_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ metric:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ site_composite:
+ index: not_analyzed
+ type: string
+interaction_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ ad_tag_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ flight_id:
+ type: integer
+ creative_id:
+ type: integer
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+ base_feature:
+ index: not_analyzed
+ type: string
+ detail:
+ index: not_analyzed
+ type: string
+ datax:
+ index: not_analyzed
+ type: string
+browser_count:
+ _all:
+ enabled: false
+ _routing:
+ path: flight_id
+ dynamic: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ cnt:
+ type: integer
+ flight_id:
+ type: integer
+ pl_test:
+ index: not_analyzed
+ type: string
+ browser_ua:
+ index: not_analyzed
+ type: string
+ tb_h:
+ type: date
+ feature:
+ index: not_analyzed
+ type: string
+parse_exception:
+ dynamic: false
+ _all:
+ enabled: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ ad_tag_id:
+ type: integer
+ cnt:
+ type: integer
+ error_code:
+ type: string
+ index: not_analyzed
+ tb_h:
+ type: date
+mountweasel:
+ dynamic: false
+ _all:
+ enabled: false
+ properties:
+ _state:
+ type: string
+ index: "no"
+ store: "yes"
+ tb_h:
+ type: date
+ cnt:
+ type: integer
View
@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+require 'httparty'
+require 'yaml'
+
+class WarmIndices < Wukong::Runner
+ include Wukong::Plugin
+
+ description <<-DESC.gsub(/^ {4}/, '').strip
+ Run this script to warm all indices in an Elasticsearch cluster.
+ This relies on the mapping settings provided in
+
+ config/mappings.yml
+
+ Run with the --debug (-d) option to see pretty printed output without
+ actually calling the Elasticsearch API.
+ DESC
+
+ class << self
+ def configure(env, prog)
+ env.define :host, default: 'localhost', flag: 'c', description: 'The hostname of the Elasticsearch cluster'
+ env.define :port, default: 9200, flag: 'p', description: 'The port number that Elasticsearch opened for web requests'
+ env.define :debug, default: false, flag: 'd', description: 'Run in debug mode'
+ env.define :indices, type: Array, flag: 'i', description: 'Comma-separated list of indices to warm. Defaults to all'
+ end
+ end
+
+ def mappings
+ @mappings ||= YAML.load File.read(File.expand_path('../../config/mappings.yml', __FILE__))
+ end
+
+ def indices
+ settings.indices || mappings.keys
+ end
+
+ def warmer_id index
+ "#{index}_warmer"
+ end
+
+ def warmer_uri index
+ "http://#{settings.host}:#{settings.port}/#{index}/_warmer/#{warmer_id(index)}?"
+ end
+
+ def sorted_query fields
+ { query: { match_all: {} }, sort: fields }
+ end
+
+ def run
+ indices.each do |index|
+ fields = mappings[index]['properties'].keys rescue abort("The provided index <#{index}> was not found in config/mapping.yml")
+ uri = warmer_uri(index)
+ body = sorted_query(fields)
+ puts "Creating warmer for index #{index}"
+ if settings.debug
+ puts MultiJson.encode(body)
+ else
+ res = HTTParty.put(uri, body: MultiJson.encode(body))
+ puts res.response
+ end
+ end
+ end
+end
+
+WarmIndices.run

0 comments on commit 5f816d8

Please sign in to comment.