Import data from a google spreadsheet (#903)

* Add google api client gem and spreadsheet service model * Add csv import from a google spreadsheet endpoint and fields in the form to paste the spreadsheet url * Delete open3 require from import_wizards_controller.rb * Add params validation before importing from a google spreadsheet * Add placeholder to spreadsheet url input and make it wider * Add google sheets api key documentation in README.md * Catch exception when importing from google sheets fails and display an error message to the user. * Add ids to clickable icons and change onclick functions to be based on those ids and not on icon classes * Fix Import Wizard forms The code had the wrong identation, so the upload controls were only shown for empty collections. This commit moves the code to where it belongs, so only the Download section changes depending on having sites or not. * Support setting Google Sheets API Key via environment So we can easily configure it on Rancher/Docker Co-authored-by: Diego <dduarte@manas.com.ar> Co-authored-by: Matias Garcia Isaia <mgarcia@manas.tech>
instedd · May 14, 2021 · 82bc51b · 82bc51b
1 parent 3a21e89
commit 82bc51b
Show file tree

Hide file tree

Showing 10 changed files with 207 additions and 15 deletions.
diff --git a/Gemfile b/Gemfile
@@ -50,6 +50,7 @@ gem 'activerecord-deprecated_finders'
 gem 'msgpack', '~> 0.7.5'
 gem 'redis'
 gem 'puma', '~> 3.11.4'
+gem 'google-api-client'
 
 group :test do
   gem 'shoulda-matchers', require: false

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -168,6 +168,8 @@ GEM
     debug_inspector (0.0.2)
     debugger-linecache (1.2.0)
     decent_exposure (2.3.2)
+    declarative (0.0.20)
+    declarative-option (0.1.0)
     devise (3.3.0)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
@@ -214,6 +216,23 @@ GEM
     global_phone (1.0.1)
     global_phone_dbgen (1.0.0)
       nokogiri (~> 1.5)
+    google-api-client (0.10.3)
+      addressable (~> 2.3)
+      googleauth (~> 0.5)
+      httpclient (~> 2.7)
+      hurley (~> 0.1)
+      memoist (~> 0.11)
+      mime-types (>= 1.6)
+      representable (~> 3.0)
+      retriable (>= 2.0, < 4.0)
+    googleauth (0.5.1)
+      faraday (~> 0.9)
+      jwt (~> 1.4)
+      logging (~> 2.0)
+      memoist (~> 0.12)
+      multi_json (~> 1.11)
+      os (~> 0.9)
+      signet (~> 0.7)
     guard (2.13.0)
       formatador (>= 0.2.4)
       listen (>= 2.7, <= 4.0)
@@ -247,6 +266,7 @@ GEM
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     httpclient (2.8.3)
+    hurley (0.2)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     ice_cube (0.12.1)
@@ -268,18 +288,24 @@ GEM
       railties (>= 3.1.0, < 5.0)
       thor (~> 0.14)
     json (1.8.6)
+    jwt (1.5.6)
     knockoutjs-rails (3.2.0)
       railties (>= 3.1, < 5)
     listen (3.0.6)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9.7)
+    little-plugger (1.1.4)
     locale (2.1.0)
     lodash-rails (2.4.1)
       railties (>= 3.1)
+    logging (2.2.2)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
     lumberjack (1.0.10)
     machinist (1.0.6)
     mail (2.6.4)
       mime-types (>= 1.16, < 4)
+    memoist (0.16.2)
     memory_profiler (0.9.6)
     method_source (0.8.2)
     mime-types (3.2.2)
@@ -316,6 +342,7 @@ GEM
       omniauth (~> 1.0)
       rack-openid (~> 1.3.1)
     orm_adapter (0.5.0)
+    os (0.9.6)
     paranoia (2.1.5)
       activerecord (~> 4.0)
     phantomjs (2.1.1.0)
@@ -379,6 +406,10 @@ GEM
     redis (3.1.0)
     redis-namespace (1.5.1)
       redis (~> 3.0, >= 3.0.4)
+    representable (3.0.4)
+      declarative (< 0.1.0)
+      declarative-option (< 0.2.0)
+      uber (< 0.2.0)
     resque (1.25.2)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
@@ -399,6 +430,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
+    retriable (3.1.2)
     rspec-core (3.4.4)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
@@ -435,6 +467,11 @@ GEM
     shellany (0.0.1)
     shoulda-matchers (2.7.0)
       activesupport (>= 3.0.0)
+    signet (0.11.0)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simplecov (0.9.0)
       docile (~> 1.1.0)
       multi_json
@@ -478,6 +515,7 @@ GEM
       polyglot (>= 0.3.1)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    uber (0.1.0)
     uglifier (2.5.3)
       execjs (>= 0.3.0)
       json (>= 1.8.0)
@@ -531,6 +569,7 @@ DEPENDENCIES
   foreman
   gettext (~> 3.1.2)
   gettext_i18n_rails_js!
+  google-api-client
   guard-jasmine (~> 2.0.6)
   haml-magic-translations
   haml-rails (~> 0.4)

diff --git a/README.md b/README.md
@@ -159,4 +159,51 @@ Resourcemap will forward any conversation with a logged user identifying them th
 
 If you don't want to use Intercom, you can simply omit `INTERCOM_APP_ID` or set it to `''`.
 
-To test the feature in development, add the `INTERCOM_APP_ID` variable and its value to the `environment` object inside the `web` service in `docker-compose.yml`.
+To test the feature in development, add the `INTERCOM_APP_ID` variable and its value to the `environment` object inside the `web` service in `docker-compose.yml`.
+
+# Upload files using Google Sheets Links
+
+## Overview
+
+Sometimes users won't upload files in the usual way (using the file explorer to select a CSV file), but by providing a Google Spread sheet link.
+At server side, link is validated and its content fetched using `Google::Apis::SheetsV4::SheetsService`.
+Finally, content of the sheet is written into a CSV file, which is stored same way as the other files (same directory and naming convention).
+Therefore, uploading a file through a Google Sheet link yields the same result as downloading the contents of the sheet as CSV and uploading that file in the usual way.
+
+## Setup
+
+Users can only uploads links that belongs to public Google Sheets. Though [Google Sheets API v4](https://developers.google.com/sheets/api/guides/authorizing) doesn't require an `OAuth 2.0 token` to authorize the requests, it does demands an `API_KEY` as a means of authentication. Therefore, in the next subsection we'll review how to create a `GOOGLE_SHEET_API_KEY` in a Project.
+
+### Obtaining a Google Sheet API KEY
+
+1. Create a Google Project or get into an existing one
+2. Navigate to `Credentials`
+3. Create a new `API_KEY` or select an existing one
+
+At this point we still have to enable our `API_KEY` obtained in step (3) to use `Google Sheets API v4`. Otherwise, if you attempt a request using the `API_KEY` to authenticate yourself (e.g try to read the content of a public spreadsheet), you'll obtain the following error:
+
+```
+{
+  "error": {
+    "code": 403,
+    "message": "Google Sheets API has not been used in project {project-id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=project-id then retry"
+    "status": "PERMISSION_DENIED",
+    "details": [
+      ...
+    ]
+  }
+}
+```
+
+4. Navigate to https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=#{project-id}, as pointed out by the error message. Don't forget to replace _project-id_ with the actual _id_ of the project.
+5. Enable `Google Sheets API v4` in your project
+6. Wait a few minutes until changes take effect
+
+At this point your `API_KEY` will be ready to authenticate `Google Sheets API v4` requests.
+
+### Setting `GOOGLE_SHEET_API_KEY`
+
+For `DEVELOPMENT`, add `GOOGLE_SHEET_API_KEY` in `settings.local.yml`.
+For `PRODUCTION`, add `GOOGLE_SHEET_API_KEY` along with the other variables set in settings.yml
+`GOOGLE_SHEET_API_KEY` is used by `SpreadsheetService` class to authenticate `Google Sheets API v4` requests.
+
diff --git a/app/controllers/import_wizards_controller.rb b/app/controllers/import_wizards_controller.rb
@@ -1,8 +1,12 @@
+require "fileutils"
+
 class ImportWizardsController < ApplicationController
   before_filter :authenticate_api_user!
   before_filter :show_properties_breadcrumb
   before_filter :authenticate_collection_admin!, only: :logs
 
+  before_action :validate_spreadsheet_params, only: [:import_csv_from_google_spreadsheet]
+
   authorize_resource :collection, decent_exposure: true
 
   expose(:import_job) { ImportJob.last_for current_user, collection }
@@ -15,16 +19,24 @@ def index
   end
 
   def upload_csv
+      import_csv_from_file(params[:file].original_filename, params[:file].read)
+  end
+
+  def import_csv_from_google_spreadsheet
     begin
-      csv = ImportWizard.import current_user, collection, params[:file].original_filename, params[:file].read
-      if csv.length == 1
-        message = "The uploaded csv is empty."
-        redirect_to adjustments_collection_import_wizard_path(collection), :notice => message
-      else
-        redirect_to adjustments_collection_import_wizard_path(collection)
-      end
-    rescue => ex
-      redirect_to collection_import_wizard_path(collection), :alert => ex.message
+      filename, path = from_google_spreadsheet(params[:spreadSheetLink])
+      file_content = File.read("#{path}/#{filename}")
+      import_csv_from_file(filename, file_content)
+    rescue Exception => e
+      message = "There was an error during the import process, please contact the site administrator."
+      redirect_to collection_import_wizard_path(collection), :alert => message
+    end
+  end
+
+  def validate_spreadsheet_params
+    if(!params.has_key?(:spreadSheetLink) or params[:spreadSheetLink] == "")
+      message = "The spreadsheet link is empty."
+      redirect_to collection_import_wizard_path(collection), :alert => message
     end
   end
 
@@ -81,4 +93,43 @@ def job_status
   def logs
     add_breadcrumb "Import wizard", collection_import_wizard_path(collection)
   end
+
+  private
+
+  def sheet_id_match(url)
+    match = url.match /^.*\/d\/(?<sheetId>.*)\/.*$/
+    match && match[:sheetId]
+  end
+
+  def import_csv_from_file(filename, file_content)
+    begin
+      csv = ImportWizard.import current_user, collection, filename, file_content
+      if csv.length == 1
+        message = "The uploaded csv is empty."
+        redirect_to adjustments_collection_import_wizard_path(collection), :notice => message
+      else
+        redirect_to adjustments_collection_import_wizard_path(collection)
+      end
+    rescue => ex
+      redirect_to collection_import_wizard_path(collection), :alert => ex.message
+    end
+  end
+
+  def from_google_spreadsheet(spread_sheet_link)
+    filename = "#{current_user.id}_#{collection.id}.csv"
+    path = "#{Rails.root}/tmp/import_wizard"
+    FileUtils.mkdir_p path
+
+    sheetId = sheet_id_match(spread_sheet_link)
+
+    rows = SpreadsheetService.get_data(sheetId)
+
+    CSV.open("#{path}/#{filename}", 'wb') do |file|
+      rows.each do |row|
+        file << row
+      end
+    end
+
+    return filename, path
+  end
 end
diff --git a/app/models/spreadsheet_service.rb b/app/models/spreadsheet_service.rb
@@ -0,0 +1,32 @@
+require 'google/apis/sheets_v4'
+
+class SpreadsheetService
+  def self.get_data(spreadsheet_id)
+    service = Google::Apis::SheetsV4::SheetsService.new
+    service.key = Settings.google_sheet_api_key
+
+    range = SpreadsheetService.get_range(spreadsheet_id)
+    begin
+      response = service.get_spreadsheet_values(spreadsheet_id, range)
+    rescue Exception => e
+      Rails.logger.error e.message + "\n" + e.backtrace.join("\n")
+      raise ActionController::BadRequest.new(), e.message()
+    end
+
+    response.values
+  end
+
+  def self.get_range(spreadsheet_id)
+    service = Google::Apis::SheetsV4::SheetsService.new
+    service.key = Settings.google_sheet_api_key
+
+    begin
+      sheet = service.get_spreadsheet(spreadsheet_id)
+      range = sheet.sheets[0].properties.title
+      range
+    rescue Exception => e
+      Rails.logger.error e.message + "\n" + e.backtrace.join("\n")
+      raise ActionController::BadRequest.new(), e.message()
+    end
+  end
+end
diff --git a/app/views/import_wizards/index.haml b/app/views/import_wizards/index.haml
@@ -1,5 +1,13 @@
 :javascript
-  $(function() { $('#upload').change(function() { $('#upload_form').submit() }); });
+  $(function() {
+    $('#upload').change(function() { $('#upload_form').submit() });
+    $("#upload_icon_csv").on("click", function(){
+      document.getElementById('upload').click();
+    });
+    $("#upload_icon_spreadsheet").on("click", function(){
+      $('#upload_form_spreadsheet').submit();
+    });
+  });
 
 = render '/tabs'
 
@@ -41,9 +49,17 @@
     -else
       = link_to _('Download a template CSV file'), sample_csv_api_collection_path(collection, format: 'csv'), class: "icon fimport black"
 
-    %form#upload_form{action: upload_csv_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data'}
-      %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token}
-      .icon.fexport.black Upload a CSV file to update multiple sites
-      %input#upload{type: :file, name: :file}
+    %div
+      %form#upload_form{action: upload_csv_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data', style: 'display: inline-block'}
+        %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token}
+        %div
+          #upload_icon_csv.icon.fexport.black Upload a CSV file to update multiple sites
+          %input#upload{type: :file, name: :file, style: 'visibility:hidden; width:1px; height: 1px'}
+      %form#upload_form_spreadsheet{action: import_csv_from_google_spreadsheet_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data', style: 'display: inline-block'}
+        %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token}
+        #upload_icon_spreadsheet.icon.syes_no.black{style: 'cursor: pointer;'} or add a spreadsheet url
+        %input{type: 'text', id: :spreadSheetLink, name: 'spreadSheetLink', placeholder: 'Enter url', style: 'width: 300px;'}
+
+
   - else
     = render '/current_snapshot_message'
diff --git a/config/routes.rb b/config/routes.rb
@@ -104,6 +104,7 @@
       resource :import_wizard, only: [] do
          get 'index'
          post 'upload_csv'
+         post 'import_csv_from_google_spreadsheet'
          get 'adjustments'
          get 'guess_columns_spec'
          post 'execute'

diff --git a/config/settings.yml b/config/settings.yml
@@ -20,3 +20,4 @@ smtp:
   enable_starttls_auto: <%= ENV['SETTINGS__SMTP__ENABLE_STARTTLS_AUTO'] %>
 
 # google_maps_key: changeme
+google_sheet_api_key: <%= ENV['SETTINGS__GOOGLE_SHEET_API_KEY'] %>
diff --git a/docker-env.template b/docker-env.template
@@ -13,6 +13,9 @@ SECRET_KEY_BASE=changeme
 # Google Maps API Key; obtain yours from https://developers.google.com/maps/documentation/javascript/get-api-key
 SETTINGS__GOOGLE_MAPS_KEY=changeme
 
+# Google Sheets API Key (see README.md on how to get it)
+SETTINGS__GOOGLE_SHEET_API_KEY=changeme
+
 # Newrelic configuration
 SETTINGS__NEWRELIC__LICENSE_KEY=
 

diff --git a/docker/settings.yml b/docker/settings.yml
@@ -20,3 +20,4 @@ smtp:
   enable_starttls_auto: <%= ENV['SETTINGS__SMTP__ENABLE_STARTTLS_AUTO'] %>
 
 google_maps_key: <%= ENV['SETTINGS__GOOGLE_MAPS_KEY'] %>
+google_sheet_api_key: <%= ENV['SETTINGS__GOOGLE_SHEET_API_KEY'] %>