Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Student Voice importer: Add importer class to automate import #2579

Merged
merged 4 commits into from Sep 3, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/config_objects/per_district.rb
Expand Up @@ -421,7 +421,7 @@ def google_email_address_mapping
# educators mistakenly set any folders or documents to have
# public permissions.
def imported_google_folder_ids(key)
if @district_key == BEDFORD
if @district_key == BEDFORD || @district_key == SOMERVILLE
json = JSON.parse(ENV.fetch('IMPORTED_GOOGLE_FOLDER_IDS_JSON', '{}'))
json.fetch(key, nil)
else
Expand Down
1 change: 1 addition & 0 deletions app/importers/helpers/data_flow.rb
Expand Up @@ -11,6 +11,7 @@ class DataFlow
MERGE_UPDATE_IGNORE_UNMARKED = 'merge_update_ignore_unmarked'
MERGE_REPLACE_ALL_WITHIN_SCOPE = 'merge_replace_all_within_scope'
MERGE_CREATE_NAIVELY = 'merge_create_naively'
MERGE_APPEND_ONLY = 'merge_append_only'

OPTION_SCHOOL_SCOPE = 'option_school_scope'
OPTION_SKIP_OLD_RECORDS = 'option_skip_old_records'
Expand Down
@@ -0,0 +1,75 @@
class StudentVoiceSurveyImporter
def self.data_flow
DataFlow.new({
importer: self.name,
source: DataFlow::SOURCE_GOOGLE_DRIVE_SHEET,
frequency: DataFlow::FREQUENCY_DAILY,
options: [],
merge: DataFlow::MERGE_APPEND_ONLY,
touches: [
StudentVoiceSurveyUpload.name,
StudentVoiceCompletedSurvey.name
],
description: 'Import student voice surveys, append-only style, by reading sheet generated from Google Form'
})
end

def initialize(options:)
@log = options.fetch(:log, STDOUT)
@time_now = options.fetch(:time_now, Time.now)
@fetcher = options.fetch(:fetcher, GoogleSheetsFetcher.new)
end

def import
log(' fetching tab...')
sheet_id = read_sheet_id_from_env()
tab = fetch_tab(sheet_id)
if tab.nil?
raise 'fetch_tab returned nil'
end

log(' creating StudentVoiceSurveyUploader...')
upload_attrs = {
file_name: "#{tab.spreadsheet_name} - #{tab.tab_name}",
uploaded_by_educator_id: read_uploaded_by_educator_id_from_env()
}
uploader = StudentVoiceSurveyUploader.new(tab.tab_csv, upload_attrs, log: @log)
log(' calling StudentVoiceSurveyUploader#create_from_text!...')
student_voice_survey_upload = uploader.create_from_text!
log(' done #create_from_text!')
log(" student_voice_survey_upload.id: #{student_voice_survey_upload.id}")
log(" student_voice_survey_upload.student_voice_completed_surveys.size: #{student_voice_survey_upload.student_voice_completed_surveys.size}")
log("StudentVoiceSurveyUploader#stats: #{uploader.stats}")

nil
end

def dry_run
raise 'Not implemented; refactor StudentVoiceSurveyUploader to enable this.'
end

private
def read_uploaded_by_educator_id_from_env
educator_login_name = ENV.fetch('STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME', '')
uploaded_by_educator_id = Educator.find_by_login_name(educator_login_name).try(:id)
raise '#read_uploaded_by_educator_id_from_env found nil' if uploaded_by_educator_id.nil?
uploaded_by_educator_id
end

def read_sheet_id_from_env
sheet_id = PerDistrict.new.imported_google_folder_ids('student_voice_survey_importer_sheet_id')
raise '#read_sheet_id_from_env found nil' if sheet_id.nil?
sheet_id
end

def fetch_tab(sheet_id)
tabs = @fetcher.get_tabs_from_sheet(sheet_id)
return nil if tabs.size != 1
tabs.first
end

def log(msg)
text = if msg.class == String then msg else JSON.pretty_generate(msg) end
@log.puts "StudentVoiceSurveyImporter: #{text}"
end
end
@@ -1,3 +1,6 @@
# This class was designed for use with a UI that uploads
# CSV files. So it is slightly different than the `Processor` classes
# and how they handle merges and updates, but similar.
class StudentVoiceSurveyUploader
def initialize(file_text, upload_attrs, options = {})
@file_text = file_text
Expand Down Expand Up @@ -37,6 +40,16 @@ def create_from_text!
student_voice_survey_upload
end

def stats
{
created_records_count: @created_records_count,
empty_survey_count: @empty_survey_count,
invalid_row_columns_count: @invalid_row_columns_count,
invalid_student_local_id_count: @invalid_student_local_id_count,
invalid_student_lodal_ids_list: @invalid_student_lodal_ids_list
}
end

private
def create_streaming_csv
csv_transformer = StreamingCsvTransformer.new(@log, {
Expand Down Expand Up @@ -65,11 +78,12 @@ def process_row_or_nil(columns_map, raw_row, index)
return nil
end

# match student
student_id = Student.find_by_local_id(row_attrs[:student_lasid]).try(:id)
# match student (look at email, outside of whitelist)
student_lasid = read_student_lasid_from_row(raw_row, row_attrs)
student_id = Student.find_by_local_id(student_lasid).try(:id)
if student_id.nil?
@invalid_student_local_id_count += 1
@invalid_student_lodal_ids_list << row_attrs[:student_lasid]
@invalid_student_lodal_ids_list << student_lasid
return nil
end

Expand All @@ -81,14 +95,21 @@ def process_row_or_nil(columns_map, raw_row, index)
})
end

def stats
{
created_records_count: @created_records_count,
empty_survey_count: @empty_survey_count,
invalid_row_columns_count: @invalid_row_columns_count,
invalid_student_local_id_count: @invalid_student_local_id_count,
invalid_student_lodal_ids_list: @invalid_student_lodal_ids_list
}
# Read the value from `Email address` if it was collected,
# and if the username is numeric (ie, a LASID).
# This avoids errors when there are typos in the field asking
# for the LASID, but fall back to that if need be.
def read_student_lasid_from_row(raw_row, row_attrs)
if raw_row.has_key?('Email address')
student_email_address = raw_row['Email address']
email_prefix = student_email_address.split('@').try(:first).try(:trim)
if /^[0-9]+$/.match?(email_prefix)
email_prefix
end
end

# fall back
row_attrs[:student_lasid]
end

def reset_counters!
Expand Down
2 changes: 1 addition & 1 deletion spec/config_objects/per_district_spec.rb
Expand Up @@ -62,10 +62,10 @@ def for_demo
end

it 'works across districts' do
expect { for_somerville.imported_google_folder_ids('foo') }.to raise_error(Exceptions::DistrictKeyNotHandledError)
expect { for_new_bedford.imported_google_folder_ids('foo') }.to raise_error(Exceptions::DistrictKeyNotHandledError)
expect { for_demo.imported_google_folder_ids('foo') }.to raise_error(Exceptions::DistrictKeyNotHandledError)
expect(for_bedford.imported_google_folder_ids('foo')).to eq 'bar'
expect(for_somerville.imported_google_folder_ids('foo')).to eq 'bar'
end
end

Expand Down
@@ -0,0 +1,104 @@
require 'rails_helper'

RSpec.describe StudentVoiceSurveyImporter do
let!(:pals) { TestPals.create! }

def create_mock_fetcher_from_map(sheet_id_to_tab_map)
mock_fetcher = GoogleSheetsFetcher.new
allow(GoogleSheetsFetcher).to receive(:new).and_return(mock_fetcher)
sheet_id_to_tab_map.each do |sheet_id, tabs|
allow(mock_fetcher).to receive(:get_tabs_from_sheet).with(sheet_id).and_return(tabs)
end
mock_fetcher
end

def create_mock_fetcher
create_mock_fetcher_from_map({
'mock_sheet_id_A' => [GoogleSheetsFetcher::Tab.new({
spreadsheet_id: 'student-voice-survey-fall',
spreadsheet_name: 'Fall Student Voice Survey',
spreadsheet_url: 'https://example.com/student-voice-fall',
tab_id: '123456',
tab_name: 'Form responses',
tab_csv: IO.read("#{Rails.root}/spec/fixtures/student_voice_survey_v2.csv")
})]
})
end

def create_importer_with_fetcher_mocked(options = {})
log = LogHelper::FakeLog.new
fetcher = create_mock_fetcher()
importer = StudentVoiceSurveyImporter.new(options: {
log: log,
fetcher: fetcher,
time_now: pals.time_now
}.merge(options))

[importer, log]
end

it 'raises on call to dry_run' do
importer, _ = create_importer_with_fetcher_mocked(sheet_id: 'mock_sheet_id_A')
expect { importer.import }.to raise_error RuntimeError
end

context 'with empty db, and env setup for test' do
before do
StudentVoiceSurveyUpload.all.destroy_all
StudentVoiceCompletedSurvey.all.destroy_all
allow(PerDistrict).to receive(:new).and_return(PerDistrict.new(district_key: PerDistrict::SOMERVILLE))

@STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME = ENV['STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME']
@IMPORTED_GOOGLE_FOLDER_IDS_JSON = ENV['IMPORTED_GOOGLE_FOLDER_IDS_JSON']
ENV['IMPORTED_GOOGLE_FOLDER_IDS_JSON'] = '{"student_voice_survey_importer_sheet_id":"mock_sheet_id_A"}'
ENV['STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME'] = 'jodi'
end

after do
ENV['IMPORTED_GOOGLE_FOLDER_IDS_JSON'] = @IMPORTED_GOOGLE_FOLDER_IDS_JSON
ENV['STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME'] = @login_name
end

it 'raises if it cannot find uploaded_by_educator_id from env' do
ENV['STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME'] = ''
importer, _ = create_importer_with_fetcher_mocked(sheet_id: 'mock_sheet_id_A')
expect { importer.send(:read_uploaded_by_educator_id_from_env) }.to raise_error RuntimeError
end

it 'reads in value for uploaded_by_educator_id correctly' do
ENV['STUDENT_VOICE_SURVEY_IMPORTER_UPLOADED_BY_EDUCATOR_LOGIN_NAME'] = 'jodi'
importer, _ = create_importer_with_fetcher_mocked(sheet_id: 'mock_sheet_id_A')
expect(importer.send(:read_uploaded_by_educator_id_from_env)).to eq pals.shs_jodi.id
end

it 'works on happy path, with sheet_id passed, and fetcher mocked' do
importer, log = create_importer_with_fetcher_mocked(sheet_id: 'mock_sheet_id_A')

expect(StudentVoiceSurveyUpload.all.size).to eq 0
expect(StudentVoiceCompletedSurvey.all.size).to eq 0
importer.import
expect(log.output).to include(':created_records_count=>1')
expect(StudentVoiceSurveyUpload.all.size).to eq 1
expect(StudentVoiceCompletedSurvey.all.size).to eq 1
expect(StudentVoiceCompletedSurvey.pluck(:student_id)).to eq [pals.shs_freshman_mari.id]
expect(StudentVoiceSurveyUpload.pluck(:uploaded_by_educator_id)).to eq [pals.shs_jodi.id]
most_recent_survey_json = StudentVoiceCompletedSurvey.most_recent_fall_student_voice_survey(pals.shs_freshman_mari.id).as_json(except: [
:id,
:student_voice_survey_upload_id,
:created_at,
:updated_at
])
expect(most_recent_survey_json).to eq({
"student_id"=> pals.shs_freshman_mari.id,
"form_timestamp" => DateTime.parse('2018-08-12 10:28:23 +0000'),
"first_name" => "Mari",
"student_lasid" => pals.shs_freshman_mari.local_id,
"proud" => "Stole the most bases in the league this year",
"best_qualities" => "Thoughtful and think before I just open my mouth",
"activities_and_interests" => "Making podcasts, teaching my sister songs",
"nervous_or_stressed" => "when there is too much work and I don't know what to do",
"learn_best" => "are kind and explain what we need to do to get good grades"
})
end
end
end