From c9406f3db97629603aa75553f0045bb342e2e0ad Mon Sep 17 00:00:00 2001 From: Simone Carletti Date: Sun, 6 Nov 2016 11:23:31 +0100 Subject: [PATCH 1/4] Add utility to manage TLD definitions --- utils/deftld.rb | 167 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100755 utils/deftld.rb diff --git a/utils/deftld.rb b/utils/deftld.rb new file mode 100755 index 000000000..7293f6a17 --- /dev/null +++ b/utils/deftld.rb @@ -0,0 +1,167 @@ +#!/usr/bin/env ruby + +require 'optparse' +require 'pathname' +require 'json' + +class TldDefs + class TldDef + attr_accessor :name + + attr_accessor :host + attr_accessor :adapter + attr_accessor :format + attr_accessor :url + + attr_accessor :type + attr_accessor :group + attr_accessor :note + + ATTRIBUTES = { + host: :host, + adapter: :adapter, + format: :format, + url: :url, + + type: :_type, + group: :_group, + note: :_note, + } + + + # Normalizes the TLD name by appending the dot, if missing. + # + # @return [String] the normalized TLD name + def self.name(string) + string = string.to_str + string.start_with?(".") ? string : ".#{string}" + end + + def initialize(name, attributes = {}) + @name = self.class.name(name) + @attributes = {} + + update(attributes) + end + + # Updates the definition attributes. + # + # @param attributes [Hash] + # @return [void] + def update(attributes = {}) + validate(attributes) + + attributes.each do |key, value| + @attributes[ATTRIBUTES[key.to_sym]] = value + end + end + + # Validates the definitions to make sure there are no unknown attributes. + # + # @param attributes [Hash] + # @return [void] + # @raise [ArgumentError] when a definition attribute is unknown + def validate(attributes) + keys = ATTRIBUTES.keys + attributes.each do |key, _| + keys.include?(key.to_sym) or + raise ArgumentError, "Invalid attribute `#{key}`" + end + end + + # Dump the definition object into a serializable Hash. + # + # @return [Hash] the serializable hash + def dump + @attributes.reject { |_, value| value.nil? } + end + end + + def initialize(file_path) + @path = Pathname.new(file_path) + end + + def read + JSON.load(@path) + end + + def write(data) + data = Hash[data.sort] + JSON.pretty_generate(data) + end + + def count + read.count + end + + def tlds_add(*tlds, **attributes) + update do |defs| + tlds.each do |tld| + tld = TldDef.name(tld) + tlddef = TldDef.new(tld, attributes) + defs[tld] = tlddef.dump + end + end + end + + def tlds_update(*tlds, **attributes) + update do |defs| + tlds.each do |tld| + tld = TldDef.name(tld) + tlddef = TldDef.new(tld, defs[tld]).update(attributes) + defs[tld] = tlddef.dump + end + end + end + + def update + data = read + puts "#{data.count} definitions read" + yield data + write(data) + puts "#{data.count} definitions written" + data + end + + def validate + read.each do |tld, data| + TldDef.new(tld, data) + end; nil + end +end + +args = ARGV +options = {} +OptionParser.new do |opts| + opts.banner = "Usage: deftld command [options]" + + TldDefs::TldDef::ATTRIBUTES.each do |key, _| + opts.on("--#{key} [VALUE]", String, "set the #{key}") do |value| + options[key] = value + end + end + + begin + opts.parse! + rescue OptionParser::ParseError + puts opts + exit 1 + end + + if args.size.zero? + puts opts + exit 1 + end +end + +defs = TldDefs.new(File.expand_path("../../data/tld.json", __FILE__)) + +case command = args.shift +when "add" + defs.tlds_add(*args, options) +when "update" + defs.tlds_update(*args, options) +else + puts "Unknown command `#{command}`" + exit 1 +end From 69c134228789fc42565b3bac5426098e8be1d6aa Mon Sep 17 00:00:00 2001 From: Simone Carletti Date: Sun, 6 Nov 2016 14:33:39 +0100 Subject: [PATCH 2/4] Sort tlds from right-to-left Before - com - com.za ... - za After: - com ... - za - com.za --- utils/deftld.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/deftld.rb b/utils/deftld.rb index 7293f6a17..34bc28094 100755 --- a/utils/deftld.rb +++ b/utils/deftld.rb @@ -86,7 +86,7 @@ def read end def write(data) - data = Hash[data.sort] + data = Hash[data.sort_by { |tld, _| tld.split(".").reverse.join(".") }] JSON.pretty_generate(data) end From 0ffaa35a6b443177dc3a0e01bf26155429d06644 Mon Sep 17 00:00:00 2001 From: Simone Carletti Date: Sun, 6 Nov 2016 14:33:48 +0100 Subject: [PATCH 3/4] List available commands --- utils/deftld.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/utils/deftld.rb b/utils/deftld.rb index 34bc28094..f92bc7a79 100755 --- a/utils/deftld.rb +++ b/utils/deftld.rb @@ -134,6 +134,14 @@ def validate options = {} OptionParser.new do |opts| opts.banner = "Usage: deftld command [options]" + opts.separator <<~EOS + +Commands: +\tadd +\tupdate + +Options: + EOS TldDefs::TldDef::ATTRIBUTES.each do |key, _| opts.on("--#{key} [VALUE]", String, "set the #{key}") do |value| From ef826e8895c9bc10103ba2e2f49ee843e413d456 Mon Sep 17 00:00:00 2001 From: Simone Carletti Date: Sun, 6 Nov 2016 15:03:46 +0100 Subject: [PATCH 4/4] Add schema and metadata information to the definition file Since the definition file is often used in third party applications, I need a way to reference changes to the schema itself directly in the file. I adopted the approach where the will be always a _ element in the file containing metadata, such as the schema version. Consumers should always check if (and compare the version) before processing the definition list. --- .simplecov | 3 ++- lib/whois/server.rb | 1 + utils/deftld.rb | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.simplecov b/.simplecov index e4e3d392f..660a2737a 100644 --- a/.simplecov +++ b/.simplecov @@ -1,3 +1,4 @@ SimpleCov.start do add_filter "/spec/" -end \ No newline at end of file + add_filter "/utils/" +end diff --git a/lib/whois/server.rb b/lib/whois/server.rb index 362291fc7..232c3f2bd 100644 --- a/lib/whois/server.rb +++ b/lib/whois/server.rb @@ -61,6 +61,7 @@ def self.load_definitions def self.load_json(file) type = File.basename(file, File.extname(file)).to_sym JSON.load(File.read(file)).each do |allocation, settings| + next if allocation == "_" define(type, allocation, settings.delete("host"), Hash[settings.map { |k,v| [k.to_sym, v] }]) end end diff --git a/utils/deftld.rb b/utils/deftld.rb index f92bc7a79..c038e4245 100755 --- a/utils/deftld.rb +++ b/utils/deftld.rb @@ -5,6 +5,14 @@ require 'json' class TldDefs + + KEY_SCHEMA = "_".freeze + + # The current schema version for the definition file + # + # @return [String] version + SCHEMA_VERSION = "2".freeze + class TldDef attr_accessor :name @@ -86,6 +94,7 @@ def read end def write(data) + data[KEY_SCHEMA] = schema_attributes data = Hash[data.sort_by { |tld, _| tld.split(".").reverse.join(".") }] JSON.pretty_generate(data) end @@ -128,6 +137,16 @@ def validate TldDef.new(tld, data) end; nil end + + private + + def schema_attributes + { + "schema" => SCHEMA_VERSION, + "updated" => Time.now.utc, + } + end + end args = ARGV