From 82bc51b52c053b4263358e01f7a4c7cbd3ac8ac7 Mon Sep 17 00:00:00 2001 From: Diego Duarte Date: Fri, 14 May 2021 12:54:37 -0300 Subject: [PATCH] Import data from a google spreadsheet (#903) * Add google api client gem and spreadsheet service model * Add csv import from a google spreadsheet endpoint and fields in the form to paste the spreadsheet url * Delete open3 require from import_wizards_controller.rb * Add params validation before importing from a google spreadsheet * Add placeholder to spreadsheet url input and make it wider * Add google sheets api key documentation in README.md * Catch exception when importing from google sheets fails and display an error message to the user. * Add ids to clickable icons and change onclick functions to be based on those ids and not on icon classes * Fix Import Wizard forms The code had the wrong identation, so the upload controls were only shown for empty collections. This commit moves the code to where it belongs, so only the Download section changes depending on having sites or not. * Support setting Google Sheets API Key via environment So we can easily configure it on Rancher/Docker Co-authored-by: Diego Co-authored-by: Matias Garcia Isaia --- Gemfile | 1 + Gemfile.lock | 39 +++++++++++ README.md | 49 +++++++++++++- app/controllers/import_wizards_controller.rb | 69 +++++++++++++++++--- app/models/spreadsheet_service.rb | 32 +++++++++ app/views/import_wizards/index.haml | 26 ++++++-- config/routes.rb | 1 + config/settings.yml | 1 + docker-env.template | 3 + docker/settings.yml | 1 + 10 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 app/models/spreadsheet_service.rb diff --git a/Gemfile b/Gemfile index 70f751a8a..cb5272424 100644 --- a/Gemfile +++ b/Gemfile @@ -50,6 +50,7 @@ gem 'activerecord-deprecated_finders' gem 'msgpack', '~> 0.7.5' gem 'redis' gem 'puma', '~> 3.11.4' +gem 'google-api-client' group :test do gem 'shoulda-matchers', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 7f915dc8f..8db2c1bfa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -168,6 +168,8 @@ GEM debug_inspector (0.0.2) debugger-linecache (1.2.0) decent_exposure (2.3.2) + declarative (0.0.20) + declarative-option (0.1.0) devise (3.3.0) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -214,6 +216,23 @@ GEM global_phone (1.0.1) global_phone_dbgen (1.0.0) nokogiri (~> 1.5) + google-api-client (0.10.3) + addressable (~> 2.3) + googleauth (~> 0.5) + httpclient (~> 2.7) + hurley (~> 0.1) + memoist (~> 0.11) + mime-types (>= 1.6) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + googleauth (0.5.1) + faraday (~> 0.9) + jwt (~> 1.4) + logging (~> 2.0) + memoist (~> 0.12) + multi_json (~> 1.11) + os (~> 0.9) + signet (~> 0.7) guard (2.13.0) formatador (>= 0.2.4) listen (>= 2.7, <= 4.0) @@ -247,6 +266,7 @@ GEM http-cookie (1.0.3) domain_name (~> 0.5) httpclient (2.8.3) + hurley (0.2) i18n (0.9.5) concurrent-ruby (~> 1.0) ice_cube (0.12.1) @@ -268,18 +288,24 @@ GEM railties (>= 3.1.0, < 5.0) thor (~> 0.14) json (1.8.6) + jwt (1.5.6) knockoutjs-rails (3.2.0) railties (>= 3.1, < 5) listen (3.0.6) rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9.7) + little-plugger (1.1.4) locale (2.1.0) lodash-rails (2.4.1) railties (>= 3.1) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) lumberjack (1.0.10) machinist (1.0.6) mail (2.6.4) mime-types (>= 1.16, < 4) + memoist (0.16.2) memory_profiler (0.9.6) method_source (0.8.2) mime-types (3.2.2) @@ -316,6 +342,7 @@ GEM omniauth (~> 1.0) rack-openid (~> 1.3.1) orm_adapter (0.5.0) + os (0.9.6) paranoia (2.1.5) activerecord (~> 4.0) phantomjs (2.1.1.0) @@ -379,6 +406,10 @@ GEM redis (3.1.0) redis-namespace (1.5.1) redis (~> 3.0, >= 3.0.4) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) resque (1.25.2) mono_logger (~> 1.0) multi_json (~> 1.0) @@ -399,6 +430,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + retriable (3.1.2) rspec-core (3.4.4) rspec-support (~> 3.4.0) rspec-expectations (3.4.0) @@ -435,6 +467,11 @@ GEM shellany (0.0.1) shoulda-matchers (2.7.0) activesupport (>= 3.0.0) + signet (0.11.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) simplecov (0.9.0) docile (~> 1.1.0) multi_json @@ -478,6 +515,7 @@ GEM polyglot (>= 0.3.1) tzinfo (1.2.5) thread_safe (~> 0.1) + uber (0.1.0) uglifier (2.5.3) execjs (>= 0.3.0) json (>= 1.8.0) @@ -531,6 +569,7 @@ DEPENDENCIES foreman gettext (~> 3.1.2) gettext_i18n_rails_js! + google-api-client guard-jasmine (~> 2.0.6) haml-magic-translations haml-rails (~> 0.4) diff --git a/README.md b/README.md index eea718d42..faaa72d6f 100644 --- a/README.md +++ b/README.md @@ -159,4 +159,51 @@ Resourcemap will forward any conversation with a logged user identifying them th If you don't want to use Intercom, you can simply omit `INTERCOM_APP_ID` or set it to `''`. -To test the feature in development, add the `INTERCOM_APP_ID` variable and its value to the `environment` object inside the `web` service in `docker-compose.yml`. \ No newline at end of file +To test the feature in development, add the `INTERCOM_APP_ID` variable and its value to the `environment` object inside the `web` service in `docker-compose.yml`. + +# Upload files using Google Sheets Links + +## Overview + +Sometimes users won't upload files in the usual way (using the file explorer to select a CSV file), but by providing a Google Spread sheet link. +At server side, link is validated and its content fetched using `Google::Apis::SheetsV4::SheetsService`. +Finally, content of the sheet is written into a CSV file, which is stored same way as the other files (same directory and naming convention). +Therefore, uploading a file through a Google Sheet link yields the same result as downloading the contents of the sheet as CSV and uploading that file in the usual way. + +## Setup + +Users can only uploads links that belongs to public Google Sheets. Though [Google Sheets API v4](https://developers.google.com/sheets/api/guides/authorizing) doesn't require an `OAuth 2.0 token` to authorize the requests, it does demands an `API_KEY` as a means of authentication. Therefore, in the next subsection we'll review how to create a `GOOGLE_SHEET_API_KEY` in a Project. + +### Obtaining a Google Sheet API KEY + +1. Create a Google Project or get into an existing one +2. Navigate to `Credentials` +3. Create a new `API_KEY` or select an existing one + +At this point we still have to enable our `API_KEY` obtained in step (3) to use `Google Sheets API v4`. Otherwise, if you attempt a request using the `API_KEY` to authenticate yourself (e.g try to read the content of a public spreadsheet), you'll obtain the following error: + +``` +{ + "error": { + "code": 403, + "message": "Google Sheets API has not been used in project {project-id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=project-id then retry" + "status": "PERMISSION_DENIED", + "details": [ + ... + ] + } +} +``` + +4. Navigate to https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=#{project-id}, as pointed out by the error message. Don't forget to replace _project-id_ with the actual _id_ of the project. +5. Enable `Google Sheets API v4` in your project +6. Wait a few minutes until changes take effect + +At this point your `API_KEY` will be ready to authenticate `Google Sheets API v4` requests. + +### Setting `GOOGLE_SHEET_API_KEY` + +For `DEVELOPMENT`, add `GOOGLE_SHEET_API_KEY` in `settings.local.yml`. +For `PRODUCTION`, add `GOOGLE_SHEET_API_KEY` along with the other variables set in settings.yml +`GOOGLE_SHEET_API_KEY` is used by `SpreadsheetService` class to authenticate `Google Sheets API v4` requests. + \ No newline at end of file diff --git a/app/controllers/import_wizards_controller.rb b/app/controllers/import_wizards_controller.rb index e9f626fd0..1cff58946 100644 --- a/app/controllers/import_wizards_controller.rb +++ b/app/controllers/import_wizards_controller.rb @@ -1,8 +1,12 @@ +require "fileutils" + class ImportWizardsController < ApplicationController before_filter :authenticate_api_user! before_filter :show_properties_breadcrumb before_filter :authenticate_collection_admin!, only: :logs + before_action :validate_spreadsheet_params, only: [:import_csv_from_google_spreadsheet] + authorize_resource :collection, decent_exposure: true expose(:import_job) { ImportJob.last_for current_user, collection } @@ -15,16 +19,24 @@ def index end def upload_csv + import_csv_from_file(params[:file].original_filename, params[:file].read) + end + + def import_csv_from_google_spreadsheet begin - csv = ImportWizard.import current_user, collection, params[:file].original_filename, params[:file].read - if csv.length == 1 - message = "The uploaded csv is empty." - redirect_to adjustments_collection_import_wizard_path(collection), :notice => message - else - redirect_to adjustments_collection_import_wizard_path(collection) - end - rescue => ex - redirect_to collection_import_wizard_path(collection), :alert => ex.message + filename, path = from_google_spreadsheet(params[:spreadSheetLink]) + file_content = File.read("#{path}/#{filename}") + import_csv_from_file(filename, file_content) + rescue Exception => e + message = "There was an error during the import process, please contact the site administrator." + redirect_to collection_import_wizard_path(collection), :alert => message + end + end + + def validate_spreadsheet_params + if(!params.has_key?(:spreadSheetLink) or params[:spreadSheetLink] == "") + message = "The spreadsheet link is empty." + redirect_to collection_import_wizard_path(collection), :alert => message end end @@ -81,4 +93,43 @@ def job_status def logs add_breadcrumb "Import wizard", collection_import_wizard_path(collection) end + + private + + def sheet_id_match(url) + match = url.match /^.*\/d\/(?.*)\/.*$/ + match && match[:sheetId] + end + + def import_csv_from_file(filename, file_content) + begin + csv = ImportWizard.import current_user, collection, filename, file_content + if csv.length == 1 + message = "The uploaded csv is empty." + redirect_to adjustments_collection_import_wizard_path(collection), :notice => message + else + redirect_to adjustments_collection_import_wizard_path(collection) + end + rescue => ex + redirect_to collection_import_wizard_path(collection), :alert => ex.message + end + end + + def from_google_spreadsheet(spread_sheet_link) + filename = "#{current_user.id}_#{collection.id}.csv" + path = "#{Rails.root}/tmp/import_wizard" + FileUtils.mkdir_p path + + sheetId = sheet_id_match(spread_sheet_link) + + rows = SpreadsheetService.get_data(sheetId) + + CSV.open("#{path}/#{filename}", 'wb') do |file| + rows.each do |row| + file << row + end + end + + return filename, path + end end diff --git a/app/models/spreadsheet_service.rb b/app/models/spreadsheet_service.rb new file mode 100644 index 000000000..450de29d4 --- /dev/null +++ b/app/models/spreadsheet_service.rb @@ -0,0 +1,32 @@ +require 'google/apis/sheets_v4' + +class SpreadsheetService + def self.get_data(spreadsheet_id) + service = Google::Apis::SheetsV4::SheetsService.new + service.key = Settings.google_sheet_api_key + + range = SpreadsheetService.get_range(spreadsheet_id) + begin + response = service.get_spreadsheet_values(spreadsheet_id, range) + rescue Exception => e + Rails.logger.error e.message + "\n" + e.backtrace.join("\n") + raise ActionController::BadRequest.new(), e.message() + end + + response.values + end + + def self.get_range(spreadsheet_id) + service = Google::Apis::SheetsV4::SheetsService.new + service.key = Settings.google_sheet_api_key + + begin + sheet = service.get_spreadsheet(spreadsheet_id) + range = sheet.sheets[0].properties.title + range + rescue Exception => e + Rails.logger.error e.message + "\n" + e.backtrace.join("\n") + raise ActionController::BadRequest.new(), e.message() + end + end +end diff --git a/app/views/import_wizards/index.haml b/app/views/import_wizards/index.haml index d93bafa74..be378e50b 100644 --- a/app/views/import_wizards/index.haml +++ b/app/views/import_wizards/index.haml @@ -1,5 +1,13 @@ :javascript - $(function() { $('#upload').change(function() { $('#upload_form').submit() }); }); + $(function() { + $('#upload').change(function() { $('#upload_form').submit() }); + $("#upload_icon_csv").on("click", function(){ + document.getElementById('upload').click(); + }); + $("#upload_icon_spreadsheet").on("click", function(){ + $('#upload_form_spreadsheet').submit(); + }); + }); = render '/tabs' @@ -41,9 +49,17 @@ -else = link_to _('Download a template CSV file'), sample_csv_api_collection_path(collection, format: 'csv'), class: "icon fimport black" - %form#upload_form{action: upload_csv_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data'} - %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token} - .icon.fexport.black Upload a CSV file to update multiple sites - %input#upload{type: :file, name: :file} + %div + %form#upload_form{action: upload_csv_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data', style: 'display: inline-block'} + %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token} + %div + #upload_icon_csv.icon.fexport.black Upload a CSV file to update multiple sites + %input#upload{type: :file, name: :file, style: 'visibility:hidden; width:1px; height: 1px'} + %form#upload_form_spreadsheet{action: import_csv_from_google_spreadsheet_collection_import_wizard_path(collection), method: :post, enctype: 'multipart/form-data', style: 'display: inline-block'} + %input{type: :hidden, name: 'authenticity_token', value: form_authenticity_token} + #upload_icon_spreadsheet.icon.syes_no.black{style: 'cursor: pointer;'} or add a spreadsheet url + %input{type: 'text', id: :spreadSheetLink, name: 'spreadSheetLink', placeholder: 'Enter url', style: 'width: 300px;'} + + - else = render '/current_snapshot_message' diff --git a/config/routes.rb b/config/routes.rb index ae32d0ae9..39850d5e1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -104,6 +104,7 @@ resource :import_wizard, only: [] do get 'index' post 'upload_csv' + post 'import_csv_from_google_spreadsheet' get 'adjustments' get 'guess_columns_spec' post 'execute' diff --git a/config/settings.yml b/config/settings.yml index 7b878c6c8..ca9812f93 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -20,3 +20,4 @@ smtp: enable_starttls_auto: <%= ENV['SETTINGS__SMTP__ENABLE_STARTTLS_AUTO'] %> # google_maps_key: changeme +google_sheet_api_key: <%= ENV['SETTINGS__GOOGLE_SHEET_API_KEY'] %> diff --git a/docker-env.template b/docker-env.template index e4005db61..b510f6e51 100644 --- a/docker-env.template +++ b/docker-env.template @@ -13,6 +13,9 @@ SECRET_KEY_BASE=changeme # Google Maps API Key; obtain yours from https://developers.google.com/maps/documentation/javascript/get-api-key SETTINGS__GOOGLE_MAPS_KEY=changeme +# Google Sheets API Key (see README.md on how to get it) +SETTINGS__GOOGLE_SHEET_API_KEY=changeme + # Newrelic configuration SETTINGS__NEWRELIC__LICENSE_KEY= diff --git a/docker/settings.yml b/docker/settings.yml index acf282b83..15a0a1527 100644 --- a/docker/settings.yml +++ b/docker/settings.yml @@ -20,3 +20,4 @@ smtp: enable_starttls_auto: <%= ENV['SETTINGS__SMTP__ENABLE_STARTTLS_AUTO'] %> google_maps_key: <%= ENV['SETTINGS__GOOGLE_MAPS_KEY'] %> +google_sheet_api_key: <%= ENV['SETTINGS__GOOGLE_SHEET_API_KEY'] %>