Merge pull request #56 from rawfunkmaharishi/strip-bullshit

Massive refactor

lib/purdie.rb

@@ -11,15 +11,13 @@
 require 'purdie/bernard'
 require 'purdie/config'
 require 'purdie/item'
-require 'purdie/ingester'
 require 'purdie/source_list'
 require 'purdie/license_manager'
 require 'purdie/exceptions'
 require 'purdie/resolver'
 
+require 'purdie/service'
 require 'purdie/services/soundcloud'
 require 'purdie/services/flickr'
 require 'purdie/services/vimeo'
 require 'purdie/services/youtube'
-
-Dotenv.load

lib/purdie/bernard.rb

@@ -1,8 +1,6 @@
-require 'purdie'
-
 module Purdie
   class Bernard
-    attr_reader :config
+    attr_accessor :verbose
 
     def initialize
       @config = Config.instance.config
@@ -24,7 +22,7 @@ def initialize
     def fetch
       raise PurdieException.new 'No data sources specified' unless @sources
 
-      @sources.map { |s| SourceList.from_file s }.each do |source|
+      @sources.map { |s| SourceList.from_file s, @verbose }.each do |source|
         source.write
       end
     end

lib/purdie/cli.rb

@@ -14,6 +14,7 @@ def version
                   :desc => 'Specify source file to process'
     def fetch
       b = Bernard.new
+      b.verbose = true
 #      b.source_file = options[:source_file] if options[:source_file]
 
       begin

lib/purdie/exceptions.rb

@@ -1,5 +1,3 @@
-require 'purdie'
-
 module Purdie
   class CredentialsException < Exception
     attr_reader :service, :message
@@ -12,7 +10,7 @@ def initialize service, message
 
   class PurdieException < Exception
     attr_reader :message
-    
+
     def initialize message
       @message = message
     end

lib/purdie/item.rb

@@ -1,6 +1,6 @@
 module Purdie
   class Item
-    attr_reader :url, :parent_file, :datas
+    attr_reader :url, :datas
 
     def initialize url
       @url = url
@@ -20,17 +20,13 @@ def [] key
     end
 
     def service
-      c = Ingester.ingesters.select { |s| url =~ /#{s.matcher}/ }.first
+      c = Service.services.select { |s| url =~ /#{s.matcher}/ }.first
       c.new
     end
 
     def to_yaml
       Purdie.debug 'wtf'
       @datas.to_yaml
     end
-
-    def source_list= source_list
-      @parent_file = source_list.parent_file
-    end
   end
 end

lib/purdie/license_manager.rb

@@ -1,5 +1,3 @@
-require 'purdie'
-
 module Purdie
   class LicenseManager
     LOOKUPS = YAML.load File.read File.join(File.dirname(__FILE__), '..', '..', '_config/licenses.yaml')

lib/purdie/resolver.rb

@@ -3,9 +3,9 @@ class Resolver
     def self.resolve urls
       urls = [urls].flatten
       resolved = []
-      
+
       urls.each do |url|
-        service_class = Ingester.ingesters.select { |service| url =~ /#{service.matcher}/ }[0]
+        service_class = Service.services.select { |service| url =~ /#{service.matcher}/ }[0]
         resolved += service_class.resolve(url) if service_class
       end
 

lib/purdie/ingester.rb → lib/purdie/service.rb

@@ -1,27 +1,22 @@
 Dotenv.load
 
 module Purdie
-  module Ingester
-    attr_reader :config, :subconfig, :matcher
-
-    include Enumerable
-
+  module Service
     class << self
-      attr_reader :ingesters
+      attr_reader :services
     end
 
-    @ingesters = []
+    @services = []
 
     def self.included base
       # Voodoo: http://stackoverflow.com/questions/10692961/inheriting-class-methods-from-mixins
       base.extend ClassMethods
-      @ingesters.push base
+      @services.push base
     end
 
     def initialize config = nil
       config = Config.instance.config unless config
       @config = config
-      @items = []
 
       configure
     end
@@ -37,33 +32,6 @@ def configure
       end
     end
 
-    def ingest url
-      @items.push distill url
-    end
-
-    def each
-      @items.each do |item|
-        yield item
-      end
-    end
-
-    def [] key
-      @items[key]
-    end
-
-    def to_yaml
-      @items.to_yaml
-    end
-
-    def write
-      if self.any?
-        FileUtils.mkdir_p File.dirname @output_file
-        File.open @output_file, 'w' do |f|
-          f.write self.to_yaml
-        end
-      end
-    end
-
     module ClassMethods
       def resolve url
         [url]

lib/purdie/services/flickr.rb

@@ -1,4 +1,3 @@
-require 'purdie'
 require 'flickraw-cached'
 
 FlickRaw.api_key = ENV['FLICKR_API_KEY']
@@ -9,7 +8,7 @@ module Services
     class Flickr
       attr_accessor :size
 
-      include Purdie::Ingester
+      include Purdie::Service
 
       def configure
         @size = 240 # pixels
@@ -37,8 +36,7 @@ def distill url
         results['title'] = photo['title']
         results['title'] = @config['default_title'] if photo['title'] == ''
         results['date'] = photo['dates']['taken'].split(' ')[0]
-        results['photo_page'] = photo['urls'][0]['_content']
-      #  results['photo_page'] = url
+        results['photo_page'] = url
         results['photo_url'] = FlickRaw.send(Flickr.url_for_size(@size), photo)
 
         license = licenses.select {|l| l['id'] == photo['license']}[0]

lib/purdie/services/soundcloud.rb

@@ -1,10 +1,9 @@
-require 'purdie'
 require 'soundcloud'
 
 module Purdie
   module Services
     class SoundCloud
-      include Purdie::Ingester
+      include Purdie::Service
 
       def client
         @client ||= ::SoundCloud.new client_id: ENV['SOUNDCLOUD_CLIENT_ID']
@@ -19,7 +18,7 @@ def distill url
           raise CredentialsException.new self, 'duff'
         end
 
-        results = {} 
+        results = {}
         results['title'] = track['title']
         results['id'] = track['id']
         results['location'] = track['description']

lib/purdie/services/vimeo.rb

@@ -1,9 +1,7 @@
-require 'purdie'
-
 module Purdie
   module Services
     class Vimeo
-      include Purdie::Ingester
+      include Purdie::Service
 
       def get url
         @id = Purdie.get_id url

lib/purdie/services/youtube.rb

@@ -1,10 +1,9 @@
-require 'purdie'
 require 'google/api_client'
 
 module Purdie
   module Services
     class YouTube
-      include Ingester
+      include Service
 
       API_SERVICE_NAME = 'youtube'
       API_VERSION = 'v3'

lib/purdie/source_list.rb

@@ -1,52 +1,60 @@
-require 'purdie'
-
 module Purdie
   class SourceList
     include Enumerable
 
     attr_reader :items
-    attr_accessor :parent_file
+    attr_accessor :parent_file, :verbose
 
     def initialize sources
       @sources = Resolver.resolve(sources).map { |item| Item.new item }
+      @items = []
+      @bad_creds = []
+      @bad_licenses = {}
     end
 
     def [] key
       @sources[key]
     end
 
-    def each &block
+    def each
       @sources.each do |source|
         yield source
       end
     end
 
     def process
-      bad_creds = []
-      bad_licenses = {}
-      @items = []
       @sources.each do |source|
         begin
-          print "Processing #{source.url}..."
+          print "Processing #{source.url}... " if @verbose
           source.distill
-          puts 'done'
+          puts 'done' if @verbose
         rescue Purdie::CredentialsException => ce
-          bad_creds.push Purdie.basename(ce.service)
-          puts 'fail'
+          @bad_creds.push Purdie.basename(ce.service)
+          puts 'fail' if @verbose
+        rescue Purdie::LicenseException => le
+          @bad_licenses[Purdie.basename le.service].push le.name rescue @bad_licenses[Purdie.basename le.service] = [le.name]
+          puts 'fail' if @verbose
         end
         @items.push source
 
-        if bad_creds.any?
-          raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{bad_creds.uniq.join ', '}"
-        end
+        bad_credentials?
+        bad_licenses?
+      end
+    end
 
-        if bad_licenses.any?
-          bad = bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
-          message = "Unknown licenses: #{bad}"
-          message += "\n"
-          message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
-          raise Purdie::PurdieException.new message
-        end
+    def bad_credentials?
+      if @bad_creds.any?
+        raise Purdie::CredentialsException.new self, "Missing or duff credentials for: #{@bad_creds.uniq.join ', '}"
+      end
+    end
+
+    def bad_licenses?
+      if @bad_licenses.any?
+        bad = @bad_licenses.map { |k,v| "#{k}: #{v.uniq.join ', '}" }.join '; '
+        message = "Unknown licenses: #{bad}"
+        message += "\n"
+        message += 'Please consider adding the details for these licenses at https://github.com/rawfunkmaharishi/purdie/blob/master/_config/licenses.yaml'
+        raise Purdie::PurdieException.new message
       end
     end
 
@@ -68,10 +76,10 @@ def output_file
       File.join ['_data', base.join('.')]
     end
 
-    def self.from_file source_file
+    def self.from_file source_file, verbose = nil
       sl = SourceList.new File.readlines(source_file).map { |l| l.strip }.select { |i| i !~ /^#/ }
       sl.parent_file = source_file
-
+      sl.verbose = verbose
       sl
     end
   end

lib/purdie/version.rb

@@ -1,3 +1,3 @@
 module Purdie
-  VERSION = "0.0.13"
+  VERSION = "0.1.0"
 end

spec/bernard_spec.rb → spec/purdie/bernard_spec.rb

@@ -2,6 +2,12 @@
 
 module Purdie
   describe Bernard do
+    it 'does not throw a fit when initialized with no _sources dir' do
+      FileUtils.rmdir File.join(File.dirname(__FILE__), '..', '..', '_sources')
+      b = Bernard.new
+      expect {b.fetch}.to raise_error
+    end
+
     it 'processes files', :vcr do
       FileUtils.cp 'spec/support/fixtures/soundcloud.sounds', '_sources/'
       FileUtils.cp 'spec/support/fixtures/vimeo.vids', '_sources/'

spec/item_spec.rb → spec/purdie/item_spec.rb

@@ -27,12 +27,6 @@ module Purdie
       )
     end
 
-    it 'knows what file it came from' do
-      sl = instance_double('SourceList', parent_file: 'some/path/bernard.purdie')
-      i.source_list = sl
-      expect(i.parent_file).to eq 'some/path/bernard.purdie'
-    end
-
     it 'knows what service it belongs to' do
       expect(i.service.class).to eq Purdie::Services::SoundCloud
     end

spec/services/flickr_spec.rb → spec/purdie/services/flickr_spec.rb

@@ -35,8 +35,8 @@ module Services
       end
 
       it 'ingests data for a photo without a specific photographer tag', :vcr do
-        @f.ingest 'https://www.flickr.com/photos/pikesley/16649739916/'
-        expect(@f[0].to_yaml).to match /photographer: sam/
+        dist = @f.distill 'https://www.flickr.com/photos/pikesley/16649739916/'
+        expect(dist.to_yaml).to match /photographer: sam/
       end
 
       it 'falls back to the default photographer name', :vcr do
@@ -76,8 +76,8 @@ module Services
 
       context 'do the local config lookups' do
         it 'looks up the default title', :vcr do
-          @f.ingest 'https://www.flickr.com/photos/cluttercup/16393865760/'
-          expect(@f[0]['title']).to eq 'Raw Funk Maharishi'
+          dist = @f.distill 'https://www.flickr.com/photos/cluttercup/16393865760/'
+          expect(dist['title']).to eq 'Raw Funk Maharishi'
         end
       end
     end