Import data from a google spreadsheet

Gemfile

@@ -50,6 +50,7 @@ gem 'activerecord-deprecated_finders'
 gem 'msgpack', '~> 0.7.5'
 gem 'redis'
 gem 'puma', '~> 3.11.4'
+gem 'google-api-client'
 
 group :test do
   gem 'shoulda-matchers', require: false

Gemfile.lock

@@ -168,6 +168,8 @@ GEM
     debug_inspector (0.0.2)
     debugger-linecache (1.2.0)
     decent_exposure (2.3.2)
+    declarative (0.0.20)
+    declarative-option (0.1.0)
     devise (3.3.0)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
@@ -214,6 +216,23 @@ GEM
     global_phone (1.0.1)
     global_phone_dbgen (1.0.0)
       nokogiri (~> 1.5)
+    google-api-client (0.10.3)
+      addressable (~> 2.3)
+      googleauth (~> 0.5)
+      httpclient (~> 2.7)
+      hurley (~> 0.1)
+      memoist (~> 0.11)
+      mime-types (>= 1.6)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+    googleauth (0.5.1)
+      faraday (~> 0.9)
+      jwt (~> 1.4)
+      logging (~> 2.0)
+      memoist (~> 0.12)
+      multi_json (~> 1.11)
+      os (~> 0.9)
+      signet (~> 0.7)
     guard (2.13.0)
       formatador (>= 0.2.4)
       listen (>= 2.7, <= 4.0)
@@ -247,6 +266,7 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     httpclient (2.8.3)
+    hurley (0.2)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     ice_cube (0.12.1)
@@ -268,18 +288,24 @@ GEM
       railties (>= 3.1.0, < 5.0)
       thor (~> 0.14)
     json (1.8.6)
+    jwt (1.5.6)
     knockoutjs-rails (3.2.0)
       railties (>= 3.1, < 5)
     listen (3.0.6)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9.7)
+    little-plugger (1.1.4)
     locale (2.1.0)
     lodash-rails (2.4.1)
       railties (>= 3.1)
+    logging (2.2.2)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
     lumberjack (1.0.10)
     machinist (1.0.6)
     mail (2.6.4)
       mime-types (>= 1.16, < 4)
+    memoist (0.16.2)
     memory_profiler (0.9.6)
     method_source (0.8.2)
     mime-types (3.2.2)
@@ -316,6 +342,7 @@ GEM
       omniauth (~> 1.0)
       rack-openid (~> 1.3.1)
     orm_adapter (0.5.0)
+    os (0.9.6)
     paranoia (2.1.5)
       activerecord (~> 4.0)
     phantomjs (2.1.1.0)
@@ -379,6 +406,10 @@ GEM
     redis (3.1.0)
     redis-namespace (1.5.1)
       redis (~> 3.0, >= 3.0.4)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
     resque (1.25.2)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
@@ -399,6 +430,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    retriable (3.1.2)
     rspec-core (3.4.4)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
@@ -435,6 +467,11 @@ GEM
     shellany (0.0.1)
     shoulda-matchers (2.7.0)
       activesupport (>= 3.0.0)
+    signet (0.11.0)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simplecov (0.9.0)
       docile (~> 1.1.0)
       multi_json
@@ -478,6 +515,7 @@ GEM
       polyglot (>= 0.3.1)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    uber (0.1.0)
     uglifier (2.5.3)
       execjs (>= 0.3.0)
       json (>= 1.8.0)
@@ -531,6 +569,7 @@ DEPENDENCIES
   foreman
   gettext (~> 3.1.2)
   gettext_i18n_rails_js!
+  google-api-client
   guard-jasmine (~> 2.0.6)
   haml-magic-translations
   haml-rails (~> 0.4)

README.md

@@ -159,4 +159,51 @@ Resourcemap will forward any conversation with a logged user identifying them th
 
 If you don't want to use Intercom, you can simply omit `INTERCOM_APP_ID` or set it to `''`.
 
-To test the feature in development, add the `INTERCOM_APP_ID` variable and its value to the `environment` object inside the `web` service in `docker-compose.yml`.
+To test the feature in development, add the `INTERCOM_APP_ID` variable and its value to the `environment` object inside the `web` service in `docker-compose.yml`.
+
+# Upload files using Google Sheets Links
+
+## Overview
+
+Sometimes users won't upload files in the usual way (using the file explorer to select a CSV file), but by providing a Google Spread sheet link.
+At server side, link is validated and its content fetched using `Google::Apis::SheetsV4::SheetsService`.
+Finally, content of the sheet is written into a CSV file, which is stored same way as the other files (same directory and naming convention).
+Therefore, uploading a file through a Google Sheet link yields the same result as downloading the contents of the sheet as CSV and uploading that file in the usual way.
+
+## Setup
+
+Users can only uploads links that belongs to public Google Sheets. Though [Google Sheets API v4](https://developers.google.com/sheets/api/guides/authorizing) doesn't require an `OAuth 2.0 token` to authorize the requests, it does demands an `API_KEY` as a means of authentication. Therefore, in the next subsection we'll review how to create a `GOOGLE_SHEET_API_KEY` in a Project.
+
+### Obtaining a Google Sheet API KEY
+
+1. Create a Google Project or get into an existing one
+2. Navigate to `Credentials`
+3. Create a new `API_KEY` or select an existing one
+
+At this point we still have to enable our `API_KEY` obtained in step (3) to use `Google Sheets API v4`. Otherwise, if you attempt a request using the `API_KEY` to authenticate yourself (e.g try to read the content of a public spreadsheet), you'll obtain the following error:
+
+```
+{
+  "error": {
+    "code": 403,
+    "message": "Google Sheets API has not been used in project {project-id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=project-id then retry"
+    "status": "PERMISSION_DENIED",
+    "details": [
+      ...
+    ]
+  }
+}
+```
+
+4. Navigate to https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=#{project-id}, as pointed out by the error message. Don't forget to replace _project-id_ with the actual _id_ of the project.
+5. Enable `Google Sheets API v4` in your project
+6. Wait a few minutes until changes take effect
+
+At this point your `API_KEY` will be ready to authenticate `Google Sheets API v4` requests.
+
+### Setting `GOOGLE_SHEET_API_KEY`
+
+For `DEVELOPMENT`, add `GOOGLE_SHEET_API_KEY` in `settings.local.yml`.
+For `PRODUCTION`, add `GOOGLE_SHEET_API_KEY` along with the other variables set in settings.yml
+`GOOGLE_SHEET_API_KEY` is used by `SpreadsheetService` class to authenticate `Google Sheets API v4` requests.
+

app/controllers/import_wizards_controller.rb

@@ -1,8 +1,12 @@
+require "fileutils"
+
 class ImportWizardsController < ApplicationController
   before_filter :authenticate_api_user!
   before_filter :show_properties_breadcrumb
   before_filter :authenticate_collection_admin!, only: :logs
 
+  before_action :validate_spreadsheet_params, only: [:import_csv_from_google_spreadsheet]
+
   authorize_resource :collection, decent_exposure: true
 
   expose(:import_job) { ImportJob.last_for current_user, collection }
@@ -15,16 +19,24 @@ def index
   end
 
   def upload_csv
+      import_csv_from_file(params[:file].original_filename, params[:file].read)
+  end
+
+  def import_csv_from_google_spreadsheet
     begin
-      csv = ImportWizard.import current_user, collection, params[:file].original_filename, params[:file].read
-      if csv.length == 1
-        message = "The uploaded csv is empty."
-        redirect_to adjustments_collection_import_wizard_path(collection), :notice => message
-      else
-        redirect_to adjustments_collection_import_wizard_path(collection)
-      end
-    rescue => ex
-      redirect_to collection_import_wizard_path(collection), :alert => ex.message
+      filename, path = from_google_spreadsheet(params[:spreadSheetLink])
+      file_content = File.read("#{path}/#{filename}")
+      import_csv_from_file(filename, file_content)
+    rescue Exception => e
+      message = "There was an error during the import process, please contact the site administrator."
+      redirect_to collection_import_wizard_path(collection), :alert => message
+    end
+  end
+
+  def validate_spreadsheet_params
+    if(!params.has_key?(:spreadSheetLink) or params[:spreadSheetLink] == "")
+      message = "The spreadsheet link is empty."
+      redirect_to collection_import_wizard_path(collection), :alert => message
     end
   end
 
@@ -81,4 +93,43 @@ def job_status
   def logs
     add_breadcrumb "Import wizard", collection_import_wizard_path(collection)
   end
+
+  private
+
+  def sheet_id_match(url)
+    match = url.match /^.*\/d\/(?<sheetId>.*)\/.*$/
+    match && match[:sheetId]
+  end
+
+  def import_csv_from_file(filename, file_content)
+    begin
+      csv = ImportWizard.import current_user, collection, filename, file_content
+      if csv.length == 1
+        message = "The uploaded csv is empty."
+        redirect_to adjustments_collection_import_wizard_path(collection), :notice => message
+      else
+        redirect_to adjustments_collection_import_wizard_path(collection)
+      end
+    rescue => ex
+      redirect_to collection_import_wizard_path(collection), :alert => ex.message
+    end
+  end
+
+  def from_google_spreadsheet(spread_sheet_link)
+    filename = "#{current_user.id}_#{collection.id}.csv"
+    path = "#{Rails.root}/tmp/import_wizard"
+    FileUtils.mkdir_p path
+
+    sheetId = sheet_id_match(spread_sheet_link)
+
+    rows = SpreadsheetService.get_data(sheetId)
+
+    CSV.open("#{path}/#{filename}", 'wb') do |file|
+      rows.each do |row|
+        file << row
+      end
+    end
+
+    return filename, path
+  end
 end

app/models/spreadsheet_service.rb

@@ -0,0 +1,32 @@
+require 'google/apis/sheets_v4'
+
+class SpreadsheetService
+  def self.get_data(spreadsheet_id)
+    service = Google::Apis::SheetsV4::SheetsService.new
+    service.key = Settings.google_sheet_api_key
+
+    range = SpreadsheetService.get_range(spreadsheet_id)
+    begin
+      response = service.get_spreadsheet_values(spreadsheet_id, range)
+    rescue Exception => e
+      Rails.logger.error e.message + "\n" + e.backtrace.join("\n")
+      raise ActionController::BadRequest.new(), e.message()
+    end
+
+    response.values
+  end
+
+  def self.get_range(spreadsheet_id)
+    service = Google::Apis::SheetsV4::SheetsService.new
+    service.key = Settings.google_sheet_api_key
+
+    begin
+      sheet = service.get_spreadsheet(spreadsheet_id)
+      range = sheet.sheets[0].properties.title
+      range
+    rescue Exception => e
+      Rails.logger.error e.message + "\n" + e.backtrace.join("\n")
+      raise ActionController::BadRequest.new(), e.message()
+    end
+  end
+end

app/views/import_wizards/index.haml

@@ -1,5 +1,13 @@
 :javascript
-  $(function() { $('#upload').change(function() { $('#upload_form').submit() }); });
+  $(function() {
+    $('#upload').change(function() { $('#upload_form').submit() });
+    $("#upload_icon_csv").on("click", function(){
+      document.getElementById('upload').click();
+    });
+    $("#upload_icon_spreadsheet").on("click", function(){
+      $('#upload_form_spreadsheet').submit();
+    });
+  });
 
 = render '/tabs'
 
@@ -41,9 +49,17 @@
     -else
       = link_to _('Download a template CSV file'), sample_csv_api_collection_path(collection, format: 'csv'), class: "icon fimport black"
 
-    %form#upload_form{action: upload_csv_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data'}
-      %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token}
-      .icon.fexport.black Upload a CSV file to update multiple sites
-      %input#upload{type: :file, name: :file}
+    %div
+      %form#upload_form{action: upload_csv_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data', style: 'display: inline-block'}
+        %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token}
+        %div
+          #upload_icon_csv.icon.fexport.black Upload a CSV file to update multiple sites
+          %input#upload{type: :file, name: :file, style: 'visibility:hidden; width:1px; height: 1px'}
+      %form#upload_form_spreadsheet{action: import_csv_from_google_spreadsheet_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data', style: 'display: inline-block'}
+        %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token}
+        #upload_icon_spreadsheet.icon.syes_no.black{style: 'cursor: pointer;'} or add a spreadsheet url
+        %input{type: 'text', id: :spreadSheetLink, name: 'spreadSheetLink', placeholder: 'Enter url', style: 'width: 300px;'}
+
+
   - else
     = render '/current_snapshot_message'

config/routes.rb

@@ -104,6 +104,7 @@
       resource :import_wizard, only: [] do
          get 'index'
          post 'upload_csv'
+         post 'import_csv_from_google_spreadsheet'
          get 'adjustments'
          get 'guess_columns_spec'
          post 'execute'

config/settings.yml

@@ -20,3 +20,4 @@ smtp:
   enable_starttls_auto: <%= ENV['SETTINGS__SMTP__ENABLE_STARTTLS_AUTO'] %>
 
 # google_maps_key: changeme
+google_sheet_api_key: <%= ENV['SETTINGS__GOOGLE_SHEET_API_KEY'] %>

docker-env.template

@@ -13,6 +13,9 @@ SECRET_KEY_BASE=changeme
 # Google Maps API Key; obtain yours from https://developers.google.com/maps/documentation/javascript/get-api-key
 SETTINGS__GOOGLE_MAPS_KEY=changeme
 
+# Google Sheets API Key (see README.md on how to get it)
+SETTINGS__GOOGLE_SHEET_API_KEY=changeme
+
 # Newrelic configuration
 SETTINGS__NEWRELIC__LICENSE_KEY=
 

docker/settings.yml

@@ -20,3 +20,4 @@ smtp:
   enable_starttls_auto: <%= ENV['SETTINGS__SMTP__ENABLE_STARTTLS_AUTO'] %>
 
 google_maps_key: <%= ENV['SETTINGS__GOOGLE_MAPS_KEY'] %>
+google_sheet_api_key: <%= ENV['SETTINGS__GOOGLE_SHEET_API_KEY'] %>