Permalink
Browse files

Refactored Concerns::InputToOutput out of Deduplication

  • Loading branch information...
fredwu committed Apr 24, 2012
1 parent 0b413f4 commit 2a9a2cb403b6a279120df11c3b5f85d844d1366d
View
@@ -48,11 +48,10 @@ country_ips = SecurityGuard::CountryIps.new(
Returns a list of the IPs from given country and IP dictionaries. Useful for auditing IPs from higher risk nations.
```ruby
-country_ips = SecurityGuard::CountryIps.new(
+SecurityGuard::CountryIps.new(
:countries => ['Australia'],
:ips => ['4.4.4.4', '8.8.8.8', '203.206.0.1']
-)
-country_ips.result # => ['203.206.0.1']
+).result # => ['203.206.0.1']
```
You may also pass country and IP data as a line-delimited file by appending `_from_file` at the end of the attributes:
View
@@ -3,5 +3,6 @@
require 'security_guard/utils/geo_ips'
require 'security_guard/concerns/accepts_from_file'
require 'security_guard/concerns/initializable'
+require 'security_guard/concerns/input_to_output'
require 'security_guard/country_ips'
require 'security_guard/deduplication'
@@ -0,0 +1,41 @@
+module SecurityGuard
+ module Concerns
+ module InputToOutput
+ def input_to_output(args)
+ input_data, filenames = read_data_from(args[:input])
+
+ output_data = send(args[:process], input_data)
+
+ write_data_to args[:output],
+ :output_data => output_data,
+ :filenames => filenames
+ end
+
+ private
+
+ def read_data_from(folder)
+ files = Dir["#{folder}/*"].sort
+
+ raise Exception.new('Input folder is invalid or is empty.') if files.empty?
+
+ input_data = []
+ filenames = []
+
+ files.each do |file|
+ input_data << File.readlines(file).map{ |line| line.downcase.strip }
+ filenames << File.basename(file)
+ end
+
+ [input_data, filenames]
+ end
+
+ def write_data_to(folder, opts)
+ opts[:output_data].each_with_index do |array, index|
+ File.open("#{folder}/#{opts[:filenames][index]}", 'w') do |f|
+ f.puts array
+ end
+ end
+ end
+ end
+ end
+end
@@ -1,55 +1,32 @@
module SecurityGuard
class Deduplication
include Concerns::Initializable
+ include Concerns::InputToOutput
initializable :input_folder, :output_folder
- attr_accessor :source_data, :deduped_data, :filenames
-
- def initialize(args = nil)
- @source_data ||= []
- @deduped_data ||= []
- @filenames ||= []
- initializable_attrs args
- end
def process
- read_data_from input_folder
- dedupe source_data
- write_data_to output_folder
+ input_to_output :input => input_folder,
+ :output => output_folder,
+ :process => :dedupe
end
private
- def read_data_from(folder)
- files = Dir["#{folder}/*"].sort
-
- raise Exception.new('Input folder is invalid or is empty.') if files.empty?
-
- files.each do |file|
- filenames << File.basename(file)
- source_data << File.readlines(file).map{ |line| line.downcase.strip }
- end
- end
-
- def write_data_to(folder)
- deduped_data.each_with_index do |array, index|
- File.open("#{folder}/#{filenames[index]}", 'w') do |f|
- f.puts array
- end
- end
- end
-
def dedupe(data)
+ deduped_data = []
+
# start from the lowest array (in terms of dedupe priority)
data.reverse!
data_original = data.clone
data_original.each do |array|
data.shift
deduped_data << _deduped_multi(array, data)
end
+
# the top array doesn't need to be compared, just needs to be unique
deduped_data.last.uniq!
- deduped_data.reverse!
+ deduped_data.reverse
end
def _deduped_multi(target, others)
@@ -0,0 +1,74 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+class TestInputToOutput
+ include SecurityGuard::Concerns::Initializable
+ include SecurityGuard::Concerns::InputToOutput
+
+ initializable :input_folder, :output_folder
+
+ def process
+ input_to_output :input => input_folder,
+ :output => output_folder,
+ :process => :reverse
+ end
+
+ def reverse(data)
+ data.each do |lines|
+ lines.reverse!
+ end
+ data
+ end
+end
+
+describe SecurityGuard::Concerns::InputToOutput do
+ let :test_io do
+ TestInputToOutput.new(
+ :input_folder => fixture_file('dedupe_lists/'),
+ :output_folder => fixture_file('../tmp/')
+ )
+ end
+
+ let :input_data do
+ [
+ [1, 2, 3],
+ [4, 5],
+ [6, 7],
+ ]
+ end
+
+ before do
+ `rm -rf #{fixture_file('../tmp/*')}`
+ end
+
+ it 'reads data from the input folder' do
+ input_data, filenames = test_io.send :read_data_from, fixture_file('dedupe_lists/')
+ input_data.must_equal input_data
+ end
+
+ it 'records filenames' do
+ input_data, filenames = test_io.send :read_data_from, fixture_file('dedupe_lists/')
+ filenames.must_equal ['a.txt', 'b.txt', 'c.txt']
+ end
+
+ it 'writes data to the output folder' do
+ test_io.send(:write_data_to,
+ fixture_file('../tmp/'),
+ :filenames => ['a.txt', 'b.txt'],
+ :output_data => [
+ [1, 2, 3],
+ [4, 5, 6],
+ ]
+ )
+
+ File.read(fixture_file('../tmp/a.txt')).must_equal "1\n2\n3\n"
+ File.read(fixture_file('../tmp/b.txt')).must_equal "4\n5\n6\n"
+ end
+
+ it 'reads input and writes output' do
+ test_io.process
+
+ File.read(fixture_file('../tmp/a.txt')).must_equal "c@example.com\nc@example.com\nb@example.com\na@example.com\n"
+ File.read(fixture_file('../tmp/b.txt')).must_equal "d@example.com\nb@example.com\n"
+ File.read(fixture_file('../tmp/c.txt')).must_equal "e@example.com\nd@example.com\nc@example.com\n"
+ end
+end
@@ -20,28 +20,6 @@
`rm -rf #{fixture_file('../tmp/*')}`
end
- it 'reads data from the input folder' do
- dedupe.send :read_data_from, fixture_file('dedupe_lists/')
- dedupe.source_data.must_equal source_data
- end
-
- it 'records filenames' do
- dedupe.send :read_data_from, fixture_file('dedupe_lists/')
- dedupe.send(:filenames).must_equal ['a.txt', 'b.txt', 'c.txt']
- end
-
- it 'writes data to the output folder' do
- dedupe.send :filenames=, ['a.txt', 'b.txt']
- dedupe.deduped_data = [
- [1, 2, 3],
- [4, 5, 6],
- ]
- dedupe.send :write_data_to, fixture_file('../tmp/')
-
- File.read(fixture_file('../tmp/a.txt')).must_equal "1\n2\n3\n"
- File.read(fixture_file('../tmp/b.txt')).must_equal "4\n5\n6\n"
- end
-
it 'reads input and writes output' do
dedupe.process
@@ -68,8 +46,7 @@
end
it 'dedupes the source data' do
- dedupe.send :dedupe, source_data
- dedupe.deduped_data.must_equal [
+ dedupe.send(:dedupe, source_data).must_equal [
['a@example.com', 'b@example.com', 'c@example.com'],
['d@example.com'],
['e@example.com'],

0 comments on commit 2a9a2cb

Please sign in to comment.