Merge pull request #35517 from code-dot-org/ha/cr-process-state

CRv2: Process state attribute

dashboard/app/models/contact_rollups_processed.rb

@@ -74,6 +74,7 @@ def self.import_from_raw_table(batch_size = DEFAULT_BATCH_SIZE)
         processed_contact_data.merge! extract_forms_submitted(contact_data)
         processed_contact_data.merge! extract_form_roles(contact_data)
         processed_contact_data.merge! extract_roles(contact_data)
+        processed_contact_data.merge! extract_state(contact_data)
         processed_contact_data.merge! extract_updated_at(contact_data)
         valid_contacts += 1
       rescue StandardError
@@ -301,6 +302,32 @@ def self.extract_form_roles(contact_data)
     return uniq_form_roles.blank? ? {} : {form_roles: uniq_form_roles}
   end
 
+  def self.extract_state(contact_data)
+    # The priority is: school state > user state > form geo state
+    # US state in schools table is in abbreviation, must convert it back to state name.
+    school_state = extract_field_latest_value contact_data, 'dashboard.schools', 'state'
+    if school_state
+      state_name = get_us_state_from_abbr(school_state, true) || school_state
+      return {state: state_name}
+    end
+
+    user_state = extract_field_latest_value contact_data, 'dashboard.users', 'state'
+    return {state: user_state} if user_state
+
+    form_geo_state = extract_field_latest_value contact_data, 'pegasus.form_geos', 'state'
+    form_geo_state.nil? ? {} : {state: form_geo_state}
+  end
+
+  # Extract the latest value of a field in a source table from contact data.
+  # @param contact_data [Hash] output of the +parse_contact_data+ method
+  # @param table [String]
+  # @param field [String]
+  # @return the latest value or nil if no value exists
+  def self.extract_field_latest_value(contact_data, table, field)
+    values = contact_data.dig(table, field)
+    values.blank? ? nil : values.max_by {|value| value['data_updated_at']}['value']
+  end
+
   # Extracts values of a field in a source table from contact data.
   #
   # @param contact_data [Hash] compiled data from multiple source tables.
@@ -309,6 +336,7 @@ def self.extract_form_roles(contact_data)
   # @param field [String]
   # @return [Array, nil] an array of values, or nil if the field or table
   #   does not exist in the contact_data.
+  # TODO: returns empty array instead of nil. They both communicate the same thing
   def self.extract_field(contact_data, table, field)
     return nil unless contact_data.key?(table) && contact_data[table].key?(field)
     contact_data.dig(table, field).map {|item| item['value']}

dashboard/test/models/contact_rollups_pardot_memory_test.rb

@@ -125,6 +125,7 @@ class ContactRollupsPardotMemoryTest < ActiveSupport::TestCase
         'forms_submitted' => 'Census,Petition',
         'form_roles' => 'engineer,teacher',
         'roles' => 'Form Submitter',
+        'state' => 'Washington',
       }
     refute ContactRollupsPardotMemory.find_by_email(contact.email)
     PardotV2.expects(:submit_batch_request).once.returns([])
@@ -142,6 +143,7 @@ class ContactRollupsPardotMemoryTest < ActiveSupport::TestCase
       'db_Forms_Submitted' => 'Census,Petition',
       'db_Form_Roles' => 'engineer,teacher',
       'db_Roles_0' => 'Form Submitter',
+      'db_State' => 'Washington',
     }
     assert_equal expected_data_synced, record[:data_synced]
   end

dashboard/test/models/contact_rollups_processed_test.rb

@@ -72,6 +72,8 @@ class ContactRollupsProcessedTest < ActiveSupport::TestCase
       once.returns({})
     ContactRollupsProcessed.expects(:extract_roles).
       once.returns({})
+    ContactRollupsProcessed.expects(:extract_state).
+      once.returns({})
     ContactRollupsProcessed.expects(:extract_updated_at).
       once.returns({})
 
@@ -122,6 +124,53 @@ class ContactRollupsProcessedTest < ActiveSupport::TestCase
     end
   end
 
+  test 'extract_field_latest_value' do
+    table = 'dashboard.schools'
+    field = 'state'
+    base_time = Time.now.utc
+
+    tests = [
+      # 3 input params are: contact_data, table, field
+      {
+        input: [{}, nil, nil],
+        expected_output: nil
+      },
+      {
+        input: [
+          {
+            table => {
+              field => [{'value' => 'WA', 'data_updated_at' => base_time}]
+            }
+          },
+          table,
+          field
+        ],
+        expected_output: 'WA'
+      },
+      {
+        input: [
+          {
+            table => {
+              field => [
+                {'value' => 'CA', 'data_updated_at' => base_time - 1.day},
+                {'value' => 'IL', 'data_updated_at' => base_time},
+                {'value' => 'NY', 'data_updated_at' => base_time - 2.days}
+              ]
+            }
+          },
+          table,
+          field
+        ],
+        expected_output: 'IL'
+      }
+    ]
+
+    tests.each_with_index do |test, index|
+      output = ContactRollupsProcessed.extract_field_latest_value(*test[:input])
+      assert_equal test[:expected_output], output, "Test index #{index} failed"
+    end
+  end
+
   test 'extract_professional_learning_enrolled' do
     contact_data = {
       'dashboard.pd_enrollments' => {
@@ -305,6 +354,68 @@ class ContactRollupsProcessedTest < ActiveSupport::TestCase
     end
   end
 
+  test 'extract_state' do
+    base_time = Time.now.utc
+    form_geos_input = {
+      'pegasus.form_geos' => {
+        'state' => [
+          {'value' => 'Washington', 'data_updated_at' => base_time - 1.day},
+          {'value' => 'California', 'data_updated_at' => base_time},
+        ]
+      }
+    }
+    users_input = {
+      'dashboard.users' => {
+        'state' => [
+          {'value' => 'Texas', 'data_updated_at' => base_time - 1.day},
+          {'value' => 'Florida', 'data_updated_at' => base_time},
+        ]
+      }
+    }
+    schools_input = {
+      'dashboard.schools' => {
+        'state' => [
+          {'value' => 'IL', 'data_updated_at' => base_time},
+          {'value' => 'PA', 'data_updated_at' => base_time - 1.day},
+        ]
+      }
+    }
+
+    tests = [
+      {
+        input: {}, expected_output: {}
+      },
+      # data come from the same table, the most recent value wins
+      {
+        input: form_geos_input,
+        expected_output: {state: 'California'}
+      },
+      {
+        input: users_input,
+        expected_output: {state: 'Florida'}
+      },
+      {
+        input: schools_input,
+        expected_output: {state: 'Illinois'}
+      },
+      # users data has higher priority than form_geos data
+      {
+        input: form_geos_input.merge(users_input),
+        expected_output: {state: 'Florida'}
+      },
+      # schools data has higher priority than users data
+      {
+        input: form_geos_input.merge(users_input).merge(schools_input),
+        expected_output: {state: 'Illinois'}
+      }
+    ]
+
+    tests.each_with_index do |test, index|
+      output = ContactRollupsProcessed.extract_state test[:input]
+      assert_equal test[:expected_output], output, "Test index #{index} failed"
+    end
+  end
+
   test 'extract_hoc_organizer_years' do
     contact_data = {
       'pegasus.forms' => {
@@ -346,29 +457,6 @@ class ContactRollupsProcessedTest < ActiveSupport::TestCase
     assert_equal expected_output, output
   end
 
-  test 'extract_updated_at with valid input' do
-    base_time = Time.now.utc - 7.days
-    tests = [
-      {
-        input: {'table1' => {'last_data_updated_at' => base_time}},
-        expected_output: {updated_at: base_time}
-      },
-      {
-        input: {
-          'table1' => {'last_data_updated_at' => base_time - 1.day},
-          'table2' => {'last_data_updated_at' => base_time + 1.day},
-          'table3' => {'last_data_updated_at' => base_time},
-        },
-        expected_output: {updated_at: base_time + 1.day}
-      }
-    ]
-
-    tests.each_with_index do |test, index|
-      output = ContactRollupsProcessed.extract_updated_at(test[:input])
-      assert_equal test[:expected_output], output, "Test index #{index} failed"
-    end
-  end
-
   test 'extract_form_roles' do
     contact_data = {
       'dashboard.census_submissions' => {
@@ -394,6 +482,29 @@ class ContactRollupsProcessedTest < ActiveSupport::TestCase
     assert_equal expected_output, output
   end
 
+  test 'extract_updated_at with valid input' do
+    base_time = Time.now.utc - 7.days
+    tests = [
+      {
+        input: {'table1' => {'last_data_updated_at' => base_time}},
+        expected_output: {updated_at: base_time}
+      },
+      {
+        input: {
+          'table1' => {'last_data_updated_at' => base_time - 1.day},
+          'table2' => {'last_data_updated_at' => base_time + 1.day},
+          'table3' => {'last_data_updated_at' => base_time},
+        },
+        expected_output: {updated_at: base_time + 1.day}
+      }
+    ]
+
+    tests.each_with_index do |test, index|
+      output = ContactRollupsProcessed.extract_updated_at(test[:input])
+      assert_equal test[:expected_output], output, "Test index #{index} failed"
+    end
+  end
+
   test 'extract_updated_at with invalid input' do
     assert_raise StandardError do
       ContactRollupsProcessed.extract_updated_at({'table' => {}})

lib/test/cdo/contact_rollups/test_pardot_v2.rb

@@ -203,6 +203,7 @@ def test_convert_to_pardot_prospect_single_value_attributes
           user_id: 111,
           forms_submitted: 'Census,Petition',
           form_roles: 'engineer,teacher',
+          state: 'Washington',
         },
         expected_output: {
           email: 'test0@domain.com',
@@ -211,6 +212,7 @@ def test_convert_to_pardot_prospect_single_value_attributes
           db_Has_Teacher_Account: 'true',
           db_Forms_Submitted: 'Census,Petition',
           db_Form_Roles: 'engineer,teacher',
+          db_State: 'Washington',
         }
       },
       {