504: Remove unused data from importer, tighten some validations

app/assets/javascripts/student_profile/EdPlansPanel.js

@@ -177,7 +177,7 @@ function reformatPersons(personsText, elseValue = null) {
   if (!personsText || personsText === '') return elseValue;
   return (
     <ul style={{...styles.simpleList, listStyleType: 'circle'}}>
-      {personsText.split(/[,;]\s/).map(personText => (
+      {_.uniq(personsText.split(/[,;]\s/)).map(personText => (
         <li key={personText}>{personText}</li>
       ))}
     </ul>

app/importers/file_importers/ed_plan_accommodations_importer.rb

@@ -114,12 +114,8 @@ def match_accommodation(row)
       ed_plan_id: ed_plan_id,
       iac_oid: row[:iac_oid],
       iac_sep_oid: row[:iac_sep_oid],
-      iac_content_area: row[:iac_content_area],
-      iac_category: row[:iac_category],
-      iac_type: row[:iac_type],
       iac_description: row[:iac_description],
       iac_field: row[:iac_field],
-      iac_name: row[:iac_name],
       iac_last_modified: parsed_timestamp(row[:iac_last_modified])
     })
 

app/importers/file_importers/ed_plans_importer.rb

@@ -95,7 +95,6 @@ def match_ed_plan(row)
     })
     ed_plan.assign_attributes({
       sep_oid: row[:sep_oid],
-      sep_grade_level: row[:sep_grade_level],
       sep_status: row[:sep_status],
       sep_effective_date: parsed_date(row[:sep_effective_date]),
       sep_review_date: parsed_date(row[:sep_review_date]),

app/models/ed_plan.rb

@@ -1,7 +1,18 @@
 class EdPlan < ApplicationRecord
   belongs_to :student
-  has_many :ed_plan_accommodations
+  has_many :ed_plan_accommodations, dependent: :destroy
 
   validates :student, presence: true
   validates :sep_oid, presence: true, uniqueness: true
+  validates :sep_effective_date, presence: true
+  validates :sep_fieldd_006, presence: true
+  validates :sep_fieldd_007, presence: true
+
+  def specific_disability
+    sep_fieldd_006
+  end
+
+  def persons_responsible
+    sep_fieldd_007
+  end
 end

config/district_somerville.yml

@@ -8,6 +8,8 @@ sftp_filenames:
   FILENAME_FOR_COURSE_SECTION_IMPORT:               /home/jbreslin/data/courses_sections_export.txt
   FILENAME_FOR_STUDENTS_SECTION_ASSIGNMENT_IMPORT:  /home/jbreslin/data/student_section_assignment_export.txt
   FILENAME_FOR_EDUCATOR_SECTION_ASSIGNMENT_IMPORT:  /home/jbreslin/data/educator_section_assignment_export.txt
+  FILENAME_FOR_ED_PLAN_IMPORT:                      /home/jbreslin/data/student_ed_plan_export.txt
+  FILENAME_FOR_ED_PLAN_ACCOMMODATIONS_IMPORT:       /home/jbreslin/data/student_accommodations_export.txt
 
   # student photos, one-time batch
   FILENAME_FOR_PHOTOS_ZIP:                          /home/jbreslin/data/photos.zip

db/migrate/20181210184301_ed_plan_validations.rb

@@ -0,0 +1,15 @@
+class EdPlanValidations < ActiveRecord::Migration[5.2]
+  def change
+    # ed plans
+    remove_column :ed_plans, :sep_grade_level, :text # no data in export
+    change_column :ed_plans, :sep_effective_date, :date, null: false # date of implementation, required
+    change_column :ed_plans, :sep_fieldd_006, :text, null: false # specific disability, required
+    change_column :ed_plans, :sep_fieldd_007, :text, null: false # persons responsible, required
+
+    # ed plan accommodations
+    remove_column :ed_plan_accommodations, :iac_content_area, :text
+    remove_column :ed_plan_accommodations, :iac_category, :text
+    remove_column :ed_plan_accommodations, :iac_type, :text
+    remove_column :ed_plan_accommodations, :iac_name, :text
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2018_12_10_135737) do
+ActiveRecord::Schema.define(version: 2018_12_10_184301) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -113,12 +113,8 @@
     t.integer "ed_plan_id", null: false
     t.text "iac_oid", null: false
     t.text "iac_sep_oid", null: false
-    t.text "iac_content_area"
-    t.text "iac_category"
-    t.text "iac_type"
     t.text "iac_description"
     t.text "iac_field"
-    t.text "iac_name"
     t.datetime "iac_last_modified"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -128,9 +124,8 @@
   create_table "ed_plans", force: :cascade do |t|
     t.text "sep_oid", null: false
     t.integer "student_id", null: false
-    t.text "sep_grade_level"
     t.text "sep_status"
-    t.date "sep_effective_date"
+    t.date "sep_effective_date", null: false
     t.date "sep_review_date"
     t.date "sep_last_meeting_date"
     t.date "sep_district_signed_date"
@@ -142,8 +137,8 @@
     t.text "sep_fieldd_003"
     t.text "sep_fieldd_004"
     t.text "sep_fieldd_005"
-    t.text "sep_fieldd_006"
-    t.text "sep_fieldd_007"
+    t.text "sep_fieldd_006", null: false
+    t.text "sep_fieldd_007", null: false
     t.text "sep_fieldd_008"
     t.text "sep_fieldd_009"
     t.text "sep_fieldd_010"

spec/controllers/profile_controller_spec.rb

@@ -109,6 +109,9 @@ def make_request(educator, student_id)
       let!(:f_and_p) do
         EdPlan.create!({
           student_id: pals.healey_kindergarten_student.id,
+          sep_effective_date: Date.parse('2016-09-15'),
+          sep_fieldd_006: 'Health disability',
+          sep_fieldd_007: 'Rich Districtwide, Laura Principal, Sarah Teacher, Jon Arbuckle (parent)',
           sep_oid: 'test-sep-oid'
         })
       end
@@ -135,7 +138,6 @@ def make_request(educator, student_id)
           'id',
           'sep_oid',
           'student_id',
-          'sep_grade_level',
           'sep_status',
           'sep_effective_date',
           'sep_review_date',

spec/importers/file_importers/ed_plan_accommodations_importer_spec.rb

@@ -30,6 +30,9 @@ def mock_importer_download!(importer)
     it 'works on happy path, when ed plan has already been imported' do
       ed_plan = EdPlan.create!({
         sep_oid: 'test-sep-oid-1',
+        sep_effective_date: Date.parse('2016-09-15'),
+        sep_fieldd_006: 'Health disability',
+        sep_fieldd_007: 'Rich Districtwide, Laura Principal, Sarah Teacher, Jon Arbuckle (parent)',
         student_id: pals.healey_kindergarten_student.id
       })
       importer, matcher, log = create_importer
@@ -52,12 +55,8 @@ def mock_importer_download!(importer)
         "ed_plan_id"=>ed_plan.id,
         "iac_oid"=>"test-iac-oid-101",
         "iac_sep_oid"=>"test-sep-oid-1",
-        "iac_content_area"=>"All",
-        "iac_category"=>"Non-standard",
-        "iac_type"=>"504",
         "iac_description"=>"Student will meet with teachers before or after school to find out about any make-up work.",
         "iac_field"=>"Student, teachers",
-        "iac_name"=>"Other Non-Standard Accommodation",
         "iac_last_modified"=>Time.parse('2016-02-02 12:34:28.000000000 +0000')
       }])
     end

spec/importers/file_importers/ed_plans_fixture.csv

@@ -1,5 +1,4 @@
 "student_state_id","student_local_id","school_local_id","sep_oid","sep_status","sep_grade_level","sep_effective_date","sep_review_date","sep_last_meeting_date","sep_district_signed_date","sep_parent_signed_date","sep_end_date","sep_last_modified","sep_fieldd_001","sep_fieldd_002","sep_fieldd_003","sep_fieldd_004","sep_fieldd_005","sep_fieldd_006","sep_fieldd_007","sep_fieldd_008","sep_fieldd_009","sep_fieldd_010","sep_fieldd_011","sep_fieldd_012","sep_fieldd_013","sep_fieldd_014"
-"991111111","111111111","HEA","test-sep-oid-1","2",\N,"2015-09-15","2015-10-15",\N,\N,"2015-09-15","2015-10-15","1425823743000","These interventions strategies have been in place since 9/15/15 to 10/15/15: Garfield will be allowed to draw a picture on school assignments.","","","Team Members Present:  Rich Districtwide, Laura Principal\
-Email with 504 sent to teachers, guidance counselor, principal on 9/15/15.","Category",\N,\N,\N,\N,\N,\N,\N,\N,\N
+"991111111","111111111","HEA","test-sep-oid-1","2",\N,"2015-09-15","2015-10-15",\N,\N,"2015-09-15","2015-10-15","1425823743000","","","","","Category","Health disability","Rich Districtwide, Laura Principal",\N,\N,\N,\N,\N,\N,\N
 "993333333","333333333","WSNS","test-sep-oid-2","4",\N,"2015-08-23","2015-10-02",\N,\N,\N,"2016-02-22","1527857665000","","","","","","ADHD\
-General Anxiety Disorder","",\N,\N,\N,\N,\N,\N,\N
+General Anxiety Disorder","Ryan Rodriguez (student), Marcus Counselor (counselor)",\N,\N,\N,\N,\N,\N,\N

spec/importers/file_importers/ed_plans_importer_spec.rb

@@ -47,7 +47,6 @@ def mock_importer_download!(importer)
       expect(EdPlan.all.as_json(except: [:id, :created_at, :updated_at])).to eq([{
         "sep_oid"=>"test-sep-oid-1",
         "student_id"=>pals.healey_kindergarten_student.id,
-        "sep_grade_level"=>nil,
         "sep_status"=>"2",
         "sep_effective_date"=>Date.parse('Tue, 15 Sep 2015'),
         "sep_review_date"=>Date.parse('Thu, 15 Oct 2015'),
@@ -56,13 +55,13 @@ def mock_importer_download!(importer)
         "sep_parent_signed_date"=>Date.parse('Tue, 15 Sep 2015'),
         "sep_end_date"=>Date.parse('Thu, 15 Oct 2015'),
         "sep_last_modified"=>Time.parse('2015-03-08 14:09:03.000000000 +0000'),
-        "sep_fieldd_001"=> "These interventions strategies have been in place since 9/15/15 to 10/15/15: Garfield will be allowed to draw a picture on school assignments.",
+        "sep_fieldd_001"=>"",
         "sep_fieldd_002"=>"",
         "sep_fieldd_003"=>"",
-        "sep_fieldd_004"=>"Team Members Present:  Rich Districtwide, Laura Principal\\\nEmail with 504 sent to teachers, guidance counselor, principal on 9/15/15.",
+        "sep_fieldd_004"=>"",
         "sep_fieldd_005"=>"Category",
-        "sep_fieldd_006"=>nil,
-        "sep_fieldd_007"=>nil,
+        "sep_fieldd_006"=>"Health disability",
+        "sep_fieldd_007"=>"Rich Districtwide, Laura Principal",
         "sep_fieldd_008"=>nil,
         "sep_fieldd_009"=>nil,
         "sep_fieldd_010"=>nil,
@@ -73,7 +72,6 @@ def mock_importer_download!(importer)
       }, {
         "sep_oid"=>"test-sep-oid-2",
         "student_id"=>pals.west_eighth_ryan.id,
-        "sep_grade_level"=>nil,
         "sep_status"=>"4",
         "sep_effective_date"=>Date.parse('Sun, 23 Aug 2015'),
         "sep_review_date"=>Date.parse('Fri, 02 Oct 2015'),
@@ -88,7 +86,7 @@ def mock_importer_download!(importer)
         "sep_fieldd_004"=>"",
         "sep_fieldd_005"=>"",
         "sep_fieldd_006"=>"ADHD\\\nGeneral Anxiety Disorder",
-        "sep_fieldd_007"=>"",
+        "sep_fieldd_007"=>"Ryan Rodriguez (student), Marcus Counselor (counselor)",
         "sep_fieldd_008"=>nil,
         "sep_fieldd_009"=>nil,
         "sep_fieldd_010"=>nil,