Merge branch 'proxy'

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    instagram-crawler (0.1.1)
+    instagram-crawler (0.2.0)
       colorize (~> 0.8)
       http (~> 4.0)
       nokogiri (~> 1.8)

README.md

@@ -56,6 +56,14 @@ instagram-crawler -u <user_name> -d -a 20181120
 instagram-crawler -u <user_name> -l
 ```
 
+### Proxy
+
+`-P || --proxyname ` `-p || --port`
+
+```ruby
+instagram-crawler -u <user_name> -P http://example.com -p 1234
+```
+
 ### Help
 
 `instagram-crawler -h | --help`
@@ -70,6 +78,8 @@ options:
     -d, --download                   Download files
     -a, --after DATE                 Download files after this date (YYYYMMDD)
     -l, --log                        Generate a log file in the current directory
+    -P, --proxyname PROXYNAME        Specify proxyname of your proxy server
+    -p, --port PORT                  Specify port of your proxy server (default port: 8080)
     -v, --version                    Show the instagram-crawler version
     -h, --help                       Show this message
 ```

bin/instagram-crawler

@@ -1,9 +1,8 @@
 #!/usr/bin/env ruby
 require_relative '../lib/instagram_crawler'
-
 begin
-  raise InstagramCrawler::Errors::EnvError if ENV["sessionid"].nil?
   args = InstagramCrawler::Parser::Args.new(ARGV)
+  raise InstagramCrawler::Errors::EnvError if ENV["sessionid"].nil?
   InstagramCrawler::Logger.setting(args.log)
   InstagramCrawler::Main.run
 rescue => e

lib/instagram_crawler/config.rb

@@ -4,7 +4,8 @@ class Config
     class << self
       attr_reader :default_url, :user_name, :base_url, :base_path,
                   :log_path, :after_date, :parse_date
-      attr_accessor :download
+      attr_accessor :download, :proxyname
+      attr_writer :port
 
       def user_name=(user_name)
         @user_name = user_name
@@ -17,6 +18,10 @@ def after_date=(after_date)
         @after_date = after_date
         @parse_date = Time.parse(after_date).to_i
       end
+
+      def port
+        @port ? @port.to_i : 8080
+      end
     end
   end
 end

lib/instagram_crawler/file.rb

@@ -25,7 +25,8 @@ def download(url, dir_name, file_name)
       private
 
       def get_binary_data(url)
-        res = HTTP.get(url)
+        res = Config.proxyname ?
+          HTTP.via(Config.proxyname, Config.port).get(url) : HTTP.get(url)
         raise Errors::HttpError, "#{res.code} #{res.reason}" if res.code != 200
         res.to_s
       end

lib/instagram_crawler/parser/args.rb

@@ -20,6 +20,8 @@ def parse_args
         opts.on('-d', '--download', 'Download files') { |download| Config.download = true }
         opts.on('-a', '--after DATE', 'Download files after this date (YYYYMMDD)') { |after_date| Config.after_date = after_date }
         opts.on('-l', '--log', 'Generate a log file in the current directory') { self.log = true }
+        opts.on('-P', '--proxyname PROXYNAME', 'Specify proxyname of your proxy server') { |proxyname| Config.proxyname = proxyname }
+        opts.on('-p', '--port PORT', 'Specify port of your proxy server (default port: 8080)') { |port| Config.port = port }
         opts.on('-v', '--version', 'Show the instagram-crawler version') { puts("instagram-crawler #{InstagramCrawler::VERSION}"); exit }
         opts.on('-h', '--help', 'Show this message') { puts(opts); exit }
         opts.parse!(@args)

lib/instagram_crawler/parser/html.rb

@@ -70,7 +70,8 @@ def loop_edges(edges)
       end
 
       def get_html(url)
-        res = HTTP.get(url)
+        res = Config.proxyname ?
+          HTTP.via(Config.proxyname, Config.port).get(url) : HTTP.get(url)
         raise Errors::HttpError, "#{res.code} #{res.reason}" if res.code != 200
         res.to_s
       end

lib/instagram_crawler/parser/json.rb

@@ -47,7 +47,9 @@ def loop_edges(edges)
       end
 
       def get_json(url)
-        res = HTTP.cookies(sessionid: ENV["sessionid"]).get(url)
+        http = HTTP.cookies(sessionid: ENV["sessionid"])
+        res = Config.proxyname ?
+          http.via(Config.proxyname, Config.port).get(url) : http.get(url)
         raise Errors::HttpError, "#{res.code} #{res.reason}" if res.code != 200
         res.to_s
       end

lib/instagram_crawler/version.rb

@@ -1,3 +1,3 @@
 module InstagramCrawler
-  VERSION = "0.1.1".freeze
+  VERSION = "0.2.0".freeze
 end

spec/instagram_crawler/parser/args_spec.rb

@@ -4,22 +4,26 @@
 
   context 'when type arguments' do
     it 'should get right with short args' do
-      parsms = ['-u', 'marvel', '-a', '20180101', '-d', '-l']
+      parsms = ['-u', 'marvel', '-a', '20180101', '-d', '-l', '-P', 'http://example.com', '-p', '1234']
 
       arg = args.new(parsms)
       expect(config.user_name).to eq 'marvel'
       expect(config.download).to eq true
       expect(config.after_date).to eq '20180101'
       expect(arg.log).to eq true
+      expect(config.proxyname).to eq 'http://example.com'
+      expect(config.port).to eq 1234
     end
 
     it 'should get right with long args' do
-      parsms = ['--username', 'marvel', '--after', '20180101', '--download', '--log']
+      parsms = ['--username', 'marvel', '--after', '20180101', '--download', '--log', '--proxyname', 'http://example.com', '--port', '1234' ]
       arg = args.new(parsms)
       expect(config.user_name).to eq 'marvel'
       expect(config.download).to eq true
       expect(config.after_date).to eq '20180101'
       expect(arg.log).to eq true
+      expect(config.proxyname).to eq 'http://example.com'
+      expect(config.port).to eq 1234
     end
   end
 end