optimized price and product loads

mvonbodun · May 7, 2022 · ba5eaeb · ba5eaeb
1 parent 4d21780
commit ba5eaeb
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 12 deletions.
diff --git a/impex/src/prices.rs b/impex/src/prices.rs
@@ -24,20 +24,28 @@ pub async fn load_prices(
     let mut rdr = csv::Reader::from_reader(input);
 
     let mut price_recs: Vec<Price> = Vec::new();
+    for line in rdr.deserialize() {
+        let record: Price = line?;
+        price_recs.push(record);
+    }
+    debug!("price_recs length: {}", price_recs.len());
 
     // Build a sku_id lookup
     let sku_id_lookup = utils::create_sku_id_lookup(client, &account_name, &environment).await;
-
-    for line in rdr.deserialize() {
-        let mut record: Price = line?;
-        let sku_id = *sku_id_lookup.get(&record.ref_id).unwrap();
-        record.sku_id = Some(sku_id);
-        price_recs.push(record);
+    let mut price_recs_with_skuid: Vec<Price> = Vec::new();
+    for mut line in price_recs {
+        let sku_id = *sku_id_lookup.get(&line.ref_id).unwrap();
+        line.sku_id = Some(sku_id);
+        price_recs_with_skuid.push(line);
     }
+    debug!(
+        "price_recs_with_skuid length: {}",
+        price_recs_with_skuid.len()
+    );
 
     let lim = Arc::new(RateLimiter::direct(Quota::per_second(rate_limit)));
     // let mut bodies = stream::iter(price_recs).ratelimit_stream(&lim);
-    let bodies = stream::iter(price_recs)
+    let bodies = stream::iter(price_recs_with_skuid)
         .map(|record| {
             let client = &client;
             let url = &url;

diff --git a/impex/src/products.rs b/impex/src/products.rs
@@ -55,7 +55,7 @@ pub async fn load_products(
 
     for line in rdr.deserialize() {
         let mut record: Product = line?;
-
+        debug!("product_record: {:?}", record);
         if skip_cat_lookup == 0 {
             // look up the category name
             let cat_unique_identifier = record.category_unique_identifier.as_ref().unwrap();
@@ -71,10 +71,12 @@ pub async fn load_products(
             record.category_id = Some(*vtex_cat_id);
         }
         // Look up the brand_id
-        let brand_name = record
-            .brand_name
-            .as_ref()
-            .unwrap_or_else(|| panic!("BrandName missing in CSV for SKU Ref: {:?}", record.ref_id));
+        let brand_name = record.brand_name.as_ref().unwrap_or_else(|| {
+            panic!(
+                "BrandName missing in CSV for Product Ref: {:?}",
+                record.ref_id
+            )
+        });
         let brand_id = brand_id_lookup.get(brand_name).unwrap_or_else(|| panic!("Brand Name: {:?} not found in lookup table.  Make sure Brand Name in the BrandName column in Products.csv matches Brand Name in the Name column of the Brands.csv file.  The values are case sensitive.", record.brand_name));
         record.brand_id = Some(*brand_id);