Maintenance: Update gitignore to be more defensive about files

.gitignore

@@ -1,46 +1,94 @@
+# See https://github.com/github/gitignore for helpful suggestions.
+
+# OS X
 .DS_Store
+
+# Rails temporary files
 **.orig
 /log
 /tmp
-/sketch
-/data
-coverage
-spec/logs/logs.txt
-dump.rdb
 
-# Node.js
+# Artificact of running simplecov analysis task
+/coverage
+
+# Node.js dependencies and logs
 node_modules
 yarn-debug.log*
 yarn-error.log*
 
-# This holds configuration for testing production import scripts locally, so it shouldn't be checked into
-# source control.
+# Build artifacts from `yarn build`
+/public/build
+/public/dev
+
+# In past development setups, files were used to holds configuration for
+# testing production import scripts by running them locally and connecting to
+# production services.  Developers should avoid working directly with production
+# credentials, and should use a Heroku dyno rather than connecting to other
+# production services directly when possible.
+# This is kept here defensively in case this file exists and contains any
+# secrets.
 config/local_env.yml
+.env
 
 # Use env initializers and read ENV variables instead, no good
 # comes from storing secrets on disk (this was deprecated
 # in Rails 5.2, but keep in here preventatively anyway).
 config/secrets.yml
 
-## Environment normalisation:
-/.bundle
-# these should all be checked in to normalise the environment:
-# Gemfile.lock, .ruby-version, .ruby-gemset
-
-# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
-.rvmrc
-
-# Editor configuration
+# See README for recommended Ruby and development environment setup
+# (eg, .ruby-version etc. should be checked into source).
+# Editor configuration (Sublime is the only supported setup)
 studentinsights.sublime-workspace
-.dir-locals.el
 
-public/build
-public/dev
 
-# For qgsocksify
+# Artificat from running qgsocksify.  Defensive against ever checking this in,
+# in case it contains any secrets.
 socks.conf
 
 # Specifically for gh-pages and jekyll
 _site/
 .sass-cache/
 .jekyll-metadata
+
+
+# Exclude all potential data files, unless explicitly safelisted.
+# This is a defense against accidentally saving production data files
+# into the project folder.  Developers should avoid working with production
+# data whenever possible, and avoid ever putting production data in the 
+# project folder, but this is another layer of defense.
+#
+# These commands can be helpful for auditing:
+#
+#  $ find . -not -path './node_modules/*' -not -path './coverage/*' -type f -name "*.json"
+#  $ git status --ignored
+#  $ git clean -d # clean and remove ignored files
+# 
+*.json
+!/app.json
+!/jest.json
+!/package.json
+!/spec/importers/helpers/data_flows_for_other_importers.json
+!/spec/importers/file_importers/data_flows_fixture.json
+
+*.csv
+!spec/fixtures/*.csv
+!spec/importers/*.csv
+
+*.sql
+!/x2_export/sql/*.sql
+
+*.xml
+!/public/browserconfig.xml
+!/x2_export/xml/*.xml
+
+*.zip
+!/spec/importers/iep_import/iep-pdfs-for-test-1.zip
+!/spec/importers/iep_import/iep-pdfs-for-test-2.zip
+
+*.log
+
+*.sqlite3
+
+# The /data folder is not currently used, but was previously used for local
+# analysis of production data, so defensively ignore anything there as well.
+/data

.rubocop.yml

@@ -9,7 +9,6 @@ AllCops:
     - 'node_modules/**/*'
     - 'public/**/*'
     - 'scripts/**/*'
-    - 'sketch/**/*'
     - 'vendor/**/*'
     - 'x2_export/**/*'
 

app/importers/helpers/fuzzy_student_matcher.rb

@@ -8,7 +8,7 @@
 #   output = matcher.match_file(file_text) {|row| matcher.match_from_last_first(row['Student_Name']) }
 class FuzzyStudentMatcher
   def initialize(options = {})
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @active_students_only = options.fetch(:active_students_only, false)
     reset_counters!
   end

app/importers/helpers/simple_syncer.rb

@@ -1,7 +1,7 @@
 # experimental
 class SimpleSyncer
   def initialize(options = {})
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @syncer = options.fetch(:syncer, RecordSyncer.new(log: @log))
     @log_frequency = options.fetch(:log_frequency, 100)
   end

app/importers/homework_help_importer/homework_help_importer.rb

@@ -9,7 +9,7 @@
 class HomeworkHelpImporter
   def initialize(educator_id, options = {})
     @educator_id = educator_id
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @matcher = options.fetch(:matcher, ImportMatcher.new)
   end
 

app/importers/iep_import/iep_pdf_import_job.rb

@@ -24,7 +24,7 @@ def initialize(options = {})
 
     @time_now = options.fetch(:time_now, Time.now)
     @s3_client = options.fetch(:s3_client, Aws::S3::Client.new)
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
   end
 
   # This imports all the IEP PDFs from a zip that

app/importers/import_task.rb

@@ -15,7 +15,7 @@ def initialize(options:)
     # to skip updating any indexes after (eg, when tuning a particular job)
     @skip_index_updates = @options.fetch('skip_index_updates', false)
 
-    @log = Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT
+    @log = Rails.env.test? ? LogHelper::FakeLog.new : STDOUT
   end
 
   def connect_transform_import

app/importers/mtss_referral/mtss_referral_processor.rb

@@ -7,7 +7,7 @@
 # event_notes = rows.map {|row| EventNote.create!(row) };nil
 class MtssReferralProcessor
   def initialize(options = {})
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
 
     @matcher = ImportMatcher.new
     @fuzzy_student_matcher = FuzzyStudentMatcher.new

app/importers/reading/f_and_p_processor.rb

@@ -12,7 +12,7 @@ class FAndPProcessor
   def initialize(benchmark_date, educator_id, options = {})
     @benchmark_date = benchmark_date
     @educator_id = educator_id
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @matcher = options.fetch(:matcher, ImportMatcher.new)
   end
 

app/importers/reading/heggerty_processor.rb

@@ -14,7 +14,7 @@ def initialize(uploaded_by_educator_id, upload_filename, options = {})
       @uploaded_by_educator_id = uploaded_by_educator_id
       @upload_filename = upload_filename
 
-      @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+      @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
       @time_now = options.fetch(:time_now, Time.now)
       @matcher = options.fetch(:matcher, ImportMatcher.new)
 

app/importers/reading/mega_reading_processor.rb

@@ -14,7 +14,7 @@ class MegaReadingProcessor
   def initialize(educator_id, benchmark_school_year, options = {})
     @educator_id = educator_id
     @benchmark_school_year = benchmark_school_year
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @matcher = options.fetch(:matcher, ImportMatcher.new)
 
     # The standard 8/20/19 template has two extra rows explaining the columns,

app/importers/reading/reading_benchmark_sheets_importer.rb

@@ -20,7 +20,7 @@ def initialize(options:)
     if @school_year != 2019
       raise "aborting because of unexpected school_year; review the syncer scoping closely before running on another year"
     end
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @dry_run = options.fetch(:dry_run, false)
     @fetcher = options.fetch(:fetcher, GoogleSheetsFetcher.new(log: @log))
     @syncer = ::RecordSyncer.new(log: @log)

app/importers/student_meeting/flat_note_importer.rb

@@ -2,7 +2,7 @@
 # the survey content into `text`).
 class FlatNoteImporter
   def initialize(options = {})
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
   end
 
   # Map `parsed_rows` into `EventNote` attributes, adding in `note_title` to each note

app/importers/student_meeting/student_meeting_importer.rb

@@ -15,7 +15,7 @@ def self.data_flow
   end
 
   def initialize(options:)
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @school_year = options.fetch(:school_year, SchoolYear.to_school_year(Time.now))
     @explicit_sheet_id = options.fetch(:explicit_sheet_id, nil)
     @fetcher = options.fetch(:fetcher, GoogleSheetsFetcher.new(log: @log))

app/importers/student_meeting/survey_reader.rb

@@ -17,7 +17,7 @@ def initialize(file_text, options = {})
     @file_text = file_text
     @source_key = options[:source_key]
     @config = options[:config]
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @google_email_address_mapping = options.fetch(:google_email_address_mapping, PerDistrict.new.google_email_address_mapping)
     @matcher = ImportMatcher.new
 

app/importers/student_voice_surveys/generic_survey_processor.rb

@@ -4,7 +4,7 @@
 # See also older survey_reader.rb, this is preferred for new code.
 class GenericSurveyProcessor
   def initialize(options, &block)
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @process_row_or_nil_block = block
 
     reset_counters!

app/importers/student_voice_surveys/mid_year_survey_importer.rb

@@ -8,7 +8,7 @@
 # records = importer.create!(file_text);nil
 class MidYearSurveyImporter
   def initialize(educator_id, form_url, options = {})
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
 
     @educator_id = educator_id
     @form_url = form_url

app/importers/student_voice_surveys/q2_self_reflection_importer.rb

@@ -8,7 +8,7 @@
 # records = importer.create!(file_text);nil
 class Q2SelfReflectionImporter
   def initialize(educator_id, form_url, options = {})
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
 
     @educator_id = educator_id
     @form_url = form_url

app/importers/student_voice_surveys/student_voice_survey_uploader.rb

@@ -5,7 +5,7 @@ class StudentVoiceSurveyUploader
   def initialize(file_text, upload_attrs, options = {})
     @file_text = file_text
     @upload_attrs = upload_attrs
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @matcher = ImportMatcher.new
     reset_counters!
   end

app/importers/team_membership_import/team_membership_importer.rb

@@ -19,7 +19,7 @@ def self.data_flow
   end
 
   def initialize(options:)
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @school_year = options.fetch(:school_year, SchoolYear.to_school_year(Time.now))
     @explicit_folder_id = options.fetch(:explicit_folder_id, nil)
     @skip_explanation_rows_count = options.fetch(:skip_explanation_rows_count, 1)

app/importers/transitions/bedford_davis_services_processor.rb

@@ -20,7 +20,7 @@ def initialize(recorded_by_educator, options = {})
 
     @time_now = options.fetch(:time_now, Time.now)
     @school_year = options.fetch(:school_year, SchoolYear.to_school_year(@time_now))
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @matcher = ImportMatcher.new
   end
 

app/importers/transitions/bedford_davis_social_emotional_processor.rb

@@ -18,7 +18,7 @@ def initialize(default_educator, options = {})
       Rollbar.warn('deprecation-warning, see RestrictedNotesProcessor and migrate to `restricted_notes`')
     end
 
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
     @matcher = ImportMatcher.new
     @time_now = options.fetch(:time_now, Time.now)
     @processor = GenericSurveyProcessor.new(log: @log) do |row|

app/importers/transitions/bedford_davis_transition_notes_processor.rb

@@ -14,7 +14,7 @@
 class BedfordDavisTransitionNotesProcessor
   def initialize(educator, form_url, options = {})
     Rollbar.warn('deprecation-warning: migrate to `services_checklist` and `teacher_forms` templates')
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
 
     @educator = educator
     @form_url = form_url

app/importers/transitions/bedford_sixth_grade_transition_processor.rb

@@ -16,7 +16,7 @@
 class BedfordSixthGradeTransitionProcessor
   def initialize(educator_id, form_url, options = {})
     Rollbar.warn('deprecation-warning')
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
 
     @educator_id = educator_id
     @form_url = form_url

app/importers/transitions/bedford_teacher_transition_processor.rb

@@ -2,7 +2,7 @@
 class BedfordTeacherTransitionProcessor
   def initialize(educator, options = {})
     Rollbar.warn('deprecation-warning, see `FormToNotesProcessor` and `teacher_forms` format')
-    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
+    @log = options.fetch(:log, Rails.env.test? ? LogHelper::FakeLog.new : STDOUT)
 
     @educator = educator
     @matcher = ImportMatcher.new