Navigation Menu

Skip to content

Commit

Permalink
Modify catalog by instance methods of the CatalogGenerator
Browse files Browse the repository at this point in the history
  • Loading branch information
piroor committed Jun 28, 2014
1 parent feeca84 commit 7aefaa5
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 58 deletions.
8 changes: 2 additions & 6 deletions bin/droonga-engine-catalog-modify
Expand Up @@ -89,12 +89,8 @@ def load_source(path)
end

source_catalog = load_source(options.source_path)
source_params = generator.catalog_to_params(source_catalog)
# TODO: Reduce too much class method calls. It is not object oriented
# programming style. It just uses class as a namespace. It is
# procedure based programming style.
modified_params = Droonga::CatalogGenerator.update_params(source_params, datasets)
modified_catalog = Droonga::CatalogGenerator.generate(modified_params)
generator.load(source_catalog)
generator.modify(datasets)

def open_output(path)
if path == "-"
Expand Down
31 changes: 14 additions & 17 deletions bin/droonga-engine-join
Expand Up @@ -97,28 +97,26 @@ end


generator = Droonga::CatalogGenerator.new
dataset_params = generator.catalog_to_params(source_catalog)
dataset_params.each do |name, dataset|
next unless dataset[:hosts].include?(options.replica_source_host)

options.dataset = name
options.tag = dataset[:tag]
options.port = dataset[:port]
options.other_hosts = dataset[:hosts]
end
generator.load(source_catalog)


unless options.dataset
dataset = generator.dataset_for_host(options.replica_source_host)
unless dataset
raise "Specified source host #{options.replica_source_host} is not a " +
"member of the cluster. You must specify correct host via " +
"--replica-source-host option."
end

options.dataset = dataset.name
options.tag = dataset.tag
options.port = dataset.port
options.other_hosts = dataset.hosts

if options.copy
puts "Initializing replica..."

destination_params = Marshal.load(Marshal.dump(dataset_params))
destination_params[options.dataset][:hosts] = [options.my_host]
destination_catalog = Droonga::CatalogGenerator.generate(destination_params)
dataset.replicas.hosts = [options.my_host]
destination_catalog = generator.catalog
Droonga::SafeFileWriter.write(catalog_path, JSON.pretty_generate(destination_catalog))

sleep(1) # wait for restart
Expand All @@ -138,10 +136,9 @@ end

puts "Restarting replica..."

final_params = Marshal.load(Marshal.dump(dataset_params))
final_params[options.dataset][:hosts] += [options.my_host]
final_params[options.dataset][:hosts].uniq!
final_catalog = Droonga::CatalogGenerator.generate(final_params)
dataset.replicas.hosts = options.other_hosts + [options.my_host]
dataset.replicas.hosts.uniq!
final_catalog = generator.catalog
Droonga::SafeFileWriter.write(catalog_path, JSON.pretty_generate(final_catalog))

sleep(1) # wait for restart
Expand Down
82 changes: 47 additions & 35 deletions lib/droonga/catalog_generator.rb
Expand Up @@ -25,6 +25,8 @@ class CatalogGenerator
DEFAULT_PORT = 10031
DEFAULT_TAG = "droonga"

attr_reader :datasets

class << self
def generate(datasets_params)
generator = new
Expand All @@ -33,23 +35,6 @@ def generate(datasets_params)
end
generator.generate
end

def update_params(base_params, modifications)
params = Marshal.load(Marshal.dump(base_params))
modifications.each do |name, modification|
dataset = params[name]
dataset[:hosts] = modification[:hosts] if modification[:hosts]

if modification[:add_replica_hosts]
dataset[:hosts] += modification[:add_replica_hosts]
dataset[:hosts].uniq!
end
if modification[:remove_replica_hosts]
dataset[:hosts] -= modification[:remove_replica_hosts]
end
end
params
end
end

def initialize
Expand All @@ -70,6 +55,44 @@ def generate
}
end

def load(catalog)
catalog["datasets"].each do |name, dataset|
add_dataset(name, dataset_to_params(dataset))
end
self
end

def dataset_for_host(host)
@datasets.each do |name, dataset|
if dataset.replicas.hosts.include?(host)
return dataset
end
end
nil
end

def modify(dataset_modifications)
dataset_modifications.each do |name, modification|
dataset = @datasets[name]
next unless dataset

replicas = dataset.replicas

if modification[:hosts]
replicas.hosts = modification[:hosts]
end

if modification[:add_replica_hosts]
dataset.hosts += modification[:add_replica_hosts]
dataset.hosts.uniq!
end

if modification[:remove_replica_hosts]
dataset.hosts -= modification[:remove_replica_hosts]
end
end
end

private
def catalog_datasets
catalog_datasets = {}
Expand All @@ -80,6 +103,8 @@ def catalog_datasets
end

class Dataset
attr_reader :name

def initialize(name, options)
@name = name
@options = options
Expand All @@ -103,15 +128,15 @@ def fact

def replicas
return @options[:replicas] if @options[:replicas]
@generated_replicas ||= Replicas.new(@options).to_json
@generated_replicas ||= Replicas.new(@options)
end

def to_catalog
catalog = {
"nWorkers" => n_workers,
"plugins" => plugins,
"schema" => schema,
"replicas" => replicas,
"replicas" => replicas.to_json,
}
catalog["fact"] = fact if fact
catalog
Expand All @@ -121,6 +146,9 @@ def to_catalog
end

class Replicas
attr_accessor :hosts
attr_reader :port, :tag, :n_slices

def initialize(options={})
@hosts = options[:hosts] || DEFAULT_HOSTS
@port = options[:port]
Expand Down Expand Up @@ -188,22 +216,6 @@ def weight
end
end

public
class << self
def catalog_to_params(catalog)
new.catalog_to_params(catalog)
end
end

def catalog_to_params(catalog)
datasets = {}
catalog["datasets"].each do |name, dataset|
datasets[name] = dataset_to_params(dataset)
end
datasets
end

private
ADDRESS_MATCHER = /\A(.*):(\d+)\/([^\.]+)\.(.+)\z/

def dataset_to_params(dataset)
Expand Down

0 comments on commit 7aefaa5

Please sign in to comment.