URLからの画像取得処理を改善

Gemfile

@@ -58,6 +58,7 @@ gem 'uglifier'
 
 # Use unicorn as the web server
 gem 'unicorn'
+gem 'unicorn-rails'
 
 # To use debugger
 # gem 'ruby-debug19', :require => 'ruby-debug'
@@ -100,3 +101,13 @@ gem 'acts_as_follower', '~> 0.1.1'
 
 # 補足出来なかった例外をメール送信
 gem 'exception_notification'
+
+# 非同期処理
+gem 'resque'
+
+# リアルタイムメッセージング
+gem 'private_pub'
+gem 'thin'
+
+# 複数アプリケーションの管理
+gem 'foreman'

Gemfile.lock

@@ -82,12 +82,14 @@ GEM
       compass (>= 0.8.7)
     compass-rails (1.0.3)
       compass (>= 0.12.2, < 0.14)
+    cookiejar (0.3.0)
     coveralls (0.7.0)
       multi_json (~> 1.3)
       rest-client
       simplecov (>= 0.7)
       term-ansicolor
       thor
+    daemons (1.1.9)
     devise (3.0.1)
       bcrypt-ruby (~> 3.0)
       orm_adapter (~> 0.1)
@@ -96,7 +98,17 @@ GEM
     diff-lcs (1.2.1)
     domain_name (0.5.7)
       unf (~> 0.0.3)
+    dotenv (0.9.0)
+    em-http-request (1.1.1)
+      addressable (>= 2.3.4)
+      cookiejar
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0.beta.2)
+    em-socksify (0.3.0)
+      eventmachine (>= 1.0.0.beta.4)
     erubis (2.7.0)
+    eventmachine (1.0.3)
     everywhere (2.0.0)
       activerecord
     exception_notification (4.0.1)
@@ -112,9 +124,23 @@ GEM
       railties (>= 3.0.0)
     faraday (0.8.7)
       multipart-post (~> 1.1)
+    faye (1.0.0)
+      cookiejar (>= 0.3.0)
+      em-http-request (>= 0.3.0)
+      eventmachine (>= 0.12.0)
+      faye-websocket (>= 0.7.0)
+      multi_json (>= 1.0.0)
+      rack (>= 1.0.0)
+      websocket-driver (>= 0.3.0)
+    faye-websocket (0.7.0)
+      eventmachine (>= 0.12.0)
+      websocket-driver (>= 0.3.0)
     font-awesome-sass-rails (3.0.2.2)
       railties (>= 3.1.1)
       sass-rails (>= 3.1.1)
+    foreman (0.63.0)
+      dotenv (>= 0.7)
+      thor (>= 0.13.6)
     fssm (0.2.10)
     haml (4.0.0)
       tilt
@@ -130,6 +156,7 @@ GEM
     highline (1.6.15)
     hike (1.2.3)
     hpricot (0.8.6)
+    http_parser.rb (0.6.0.beta.2)
     httpauth (0.2.0)
     i18n (0.6.4)
     i18n_generators (1.2.1)
@@ -167,6 +194,7 @@ GEM
       ntlm-http (~> 0.1, >= 0.1.1)
       webrobots (~> 0.0, >= 0.0.9)
     mime-types (1.23)
+    mono_logger (1.1.0)
     multi_json (1.7.7)
     multipart-post (1.2.0)
     mysql2 (0.3.11)
@@ -199,12 +227,16 @@ GEM
       omniauth (~> 1.0)
     orm_adapter (0.4.0)
     polyglot (0.3.3)
+    private_pub (1.0.3)
+      faye
     rack (1.4.5)
     rack-cache (1.2)
       rack (>= 0.4)
     rack-pjax (0.7.0)
       nokogiri (~> 1.5)
       rack (~> 1.3)
+    rack-protection (1.5.1)
+      rack
     rack-ssl (1.3.3)
       rack
     rack-test (0.6.2)
@@ -228,8 +260,17 @@ GEM
     rake (10.1.0)
     rdoc (3.12.2)
       json (~> 1.4)
+    redis (3.0.5)
+    redis-namespace (1.3.1)
+      redis (~> 3.0.0)
     ref (1.0.2)
     remotipart (1.0.5)
+    resque (1.25.1)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.2)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     rspec-core (2.13.0)
@@ -255,6 +296,10 @@ GEM
       multi_json (~> 1.0)
       simplecov-html (~> 0.7.1)
     simplecov-html (0.7.1)
+    sinatra (1.4.4)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
     slim (2.0.0)
       temple (~> 0.6.5)
       tilt (~> 1.3, >= 1.3.3)
@@ -274,6 +319,10 @@ GEM
     therubyracer (0.11.4)
       libv8 (~> 3.11.8.12)
       ref
+    thin (1.6.1)
+      daemons (>= 1.0.9)
+      eventmachine (>= 1.0.0)
+      rack (>= 1.0.0)
     thor (0.18.1)
     tilt (1.4.1)
     tins (0.12.0)
@@ -291,9 +340,15 @@ GEM
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
+    unicorn-rails (1.1.0)
+      rack
+      unicorn
+    vegas (0.1.11)
+      rack (>= 1.0.0)
     warden (1.2.3)
       rack (>= 1.0)
     webrobots (0.1.0)
+    websocket-driver (0.3.0)
     will_paginate (3.0.4)
     yaml_db (0.2.3)
     zurui-sass-rails (0.0.1)
@@ -316,6 +371,7 @@ DEPENDENCIES
   exception_notification
   factory_girl_rails
   font-awesome-sass-rails
+  foreman
   heroku
   hpricot
   i18n_generators
@@ -325,15 +381,19 @@ DEPENDENCIES
   masonry-rails
   mysql2
   omniauth-facebook
+  private_pub
   rails (= 3.2.14)
   rails_admin!
+  resque
   rspec-rails (~> 2.0)
   sass-rails (~> 3.2.3)
   settingslogic
   slim-rails
   therubyracer
+  thin
   uglifier
   unicorn
+  unicorn-rails
   will_paginate (~> 3.0.3)
   yaml_db
   zurui-sass-rails

Procfile

@@ -0,0 +1,3 @@
+web: bundle exec rails server -p $PORT
+resque: bundle exec rake resque:work QUEUE='*'
+private_pub: bundle exec rackup private_pub.ru -s thin -E production

app/assets/javascripts/application.js

@@ -24,6 +24,7 @@
 //= require masonry/jquery.masonry
 //= require masonry/jquery.imagesloaded.min
 //= require masonry/jquery.infinitescroll.min
+//= require private_pub
 
 $(function () {
   $("[data-toggle='tooltip']").tooltip();

app/controllers/clips_controller.rb

@@ -2,28 +2,31 @@
 class ClipsController < ApplicationController
   before_filter :authenticate_user!, except: [ :index, :show, :get_image_tags ]
 
+  require 'will_paginate/array'
+
   # for Ajax
   def get_image_tags
-    require 'will_paginate/array'
-    require 'hpricot'
-    html = ''
-    if params[:page].blank? || params[:page].to_i <= 1
-      @clip = Clip.new(params[:clip])
-      @clip.fill_origin_entry
-      html = @clip.create_html_only_images
-      create_html_cahce_file(html)
-    else
-      html = load_html_cahce_file
+    raise unless params[:clip]
+    raise unless params[:clip][:origin_url]
+
+    @clip = Clip.new(params[:clip])
+    raise unless @clip
+
+    if not exist_html_cahce_file?
+      Resque.enqueue(ImageClipper, @clip.origin_url)
+      raise
     end
-    doc = Hpricot(html)
+
+    doc = Hpricot(load_html_cahce_file)
     @page = (doc/:img).paginate(page: params[:page], per_page: Clip.per_page)
-    if params[:page].blank? || params[:page].to_i <= 1
+    if params[:page].to_i.zero?
       @html = @page.join
-      render
     else
       @html = "<html><body><div id='container'>#{@page.map(&insert_div_tag_for_image_tag).join}</div></body></html>"
       render text: @html
     end
+  rescue
+    render nothing: true
   end
 
   def like
@@ -183,11 +186,20 @@ def create_html_cahce_file(html)
     end
   end
 
+  def exist_html_cahce_file?
+    File.exist? html_cache_file_path
+  end
+
   def load_html_cahce_file
     File.open(html_cache_file_path).read
   end
 
   def html_cache_file_path
-    File.join(Settings.html_cache_dir, "#{current_user.id.to_s}.html")
+    File.join(Settings.html_cache_dir, "#{hash_by_target_url}.html")
+  end
+
+  def hash_by_target_url
+    require 'digest/sha2'
+    Digest::SHA256.hexdigest(@clip.origin_url)
   end
 end

app/models/clip.rb

@@ -171,6 +171,7 @@ def uri_relative_path(path)
     path
   end
 
+  # TODO: 高速化
   def create_image_tag_by_image_url(url)
     require 'open-uri'
     require 'image_size'

app/views/clips/_form.html.slim

@@ -4,6 +4,8 @@
   = javascript_include_tag "dnd"
   = javascript_include_tag "bookmarklet"
 
+= subscribe_to "/cliped"
+
 #image_selecter
   h3 1. 画像をセレクト
   = form_for @clip, url: { action: :get_image_tags }, remote: true, html: { id: 'load_form' } do |f|

app/workers/image_clipper.rb

@@ -0,0 +1,57 @@
+class ImageClipper
+  @queue = :image_clippers_queue
+
+  require 'hpricot'
+
+  class << self
+    def perform(target_url)
+      @target_url = target_url
+      download_html
+      publish
+    end
+
+    private
+
+    def publish
+      PrivatePub.publish_to('/cliped', "$.get('/clips/get_image_tags', #{params_for_publish.to_json});")
+    end
+
+    def params_for_publish
+      {
+        clip: {
+          origin_url: @target_url
+        }
+      }
+    end
+
+    def publish_for_error(message)
+      PrivatePub.publish_to('/cliped', "$('#alert').text('#{message}');")
+    end
+
+    def download_html
+      clip = Clip.new(origin_url: @target_url)
+      clip.fill_origin_entry
+      html = clip.create_html_only_images
+      create_html_cahce_file(html)
+    rescue
+      publish_for_error($!.message)
+      raise
+    end
+
+    def create_html_cahce_file(html)
+      FileUtils.mkdir(Settings.html_cache_dir) unless File.exist? Settings.html_cache_dir
+      File.open(html_cache_file_path, 'w') do |file|
+        file.write(html)
+      end
+    end
+
+    def html_cache_file_path
+      File.join(Settings.html_cache_dir, "#{hash_by_target_url}.html")
+    end
+
+    def hash_by_target_url
+      require 'digest/sha2'
+      Digest::SHA256.hexdigest(@target_url)
+    end
+  end
+end

config/private_pub.yml

@@ -0,0 +1,10 @@
+development:
+  server: "http://vistlip:9292/faye"
+  secret_token: "secret"
+test:
+  server: "http://localhost:9292/faye"
+  secret_token: "secret"
+production:
+  server: "http://clipclap.org:9292/faye"
+  secret_token: "e80f7407d68f55b87adced51d1aae06c62f493048def5e86a408019cd4d04350"
+  signature_expiration: 3600 # one hour

lib/tasks/resque.rake

@@ -0,0 +1,3 @@
+require 'resque/tasks'
+
+task 'resque:setup' => :environment

private_pub.ru

@@ -0,0 +1,10 @@
+# Run with: rackup private_pub.ru -s thin -E production
+require "bundler/setup"
+require "yaml"
+require "faye"
+require "private_pub"
+
+Faye::WebSocket.load_adapter('thin')
+
+PrivatePub.load_config(File.expand_path("../config/private_pub.yml", __FILE__), ENV["RAILS_ENV"] || "development")
+run PrivatePub.faye_app