#5 Fix skelbiu-lt description can sometimes be null

zexa · Oct 12, 2021 · f3fd2b3 · f3fd2b3
1 parent 0787a4f
commit f3fd2b3
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 13 deletions.
diff --git a/skelbiu-lt-scraper/src/skelbiu_lt_listing.rs b/skelbiu-lt-scraper/src/skelbiu_lt_listing.rs
@@ -6,7 +6,7 @@ pub struct SkelbiuLtListing {
     url: String,
     id: String,
     title: String,
-    description: String,
+    description: Option<String>,
     views: String,
     updated_at: String,
     liked_amount: String,
@@ -20,7 +20,7 @@ impl SkelbiuLtListing {
         url: String,
         id: String,
         title: String,
-        description: String,
+        description: Option<String>,
         views: String,
         updated_at: String,
         liked_amount: String,
@@ -50,7 +50,7 @@ impl SkelbiuLtListing {
         &self.title
     }
 
-    pub fn get_description(&self) -> &str {
+    pub fn get_description(&self) -> &Option<String> {
         &self.description
     }
 

diff --git a/skelbiu-lt-scraper/src/skelbiu_lt_listing_scraper.rs b/skelbiu-lt-scraper/src/skelbiu_lt_listing_scraper.rs
@@ -1,6 +1,6 @@
 use crate::skelbiu_lt_listing::SkelbiuLtListing;
 use common_scraper::{ListingScraper, PotentialListing};
-use scraper::Selector;
+use scraper::{ElementRef, Selector};
 use slog::Logger;
 
 pub struct SkelbiuLtListingScraper {
@@ -75,15 +75,21 @@ impl ListingScraper<SkelbiuLtListing> for SkelbiuLtListingScraper {
                 .to_string();
             debug!(self.logger, "Found title for {}", &listing_url);
 
-            let description = html
-                .select(&self.description_selector)
-                .next()
-                .unwrap_or_else(|| panic!("Could not find description for {}", &listing_url))
-                .text()
-                .collect::<String>()
-                .trim()
-                .to_string();
-            debug!(self.logger, "Found description for {}", &listing_url);
+            let description = match html.select(&self.description_selector).next() {
+                None => {
+                    debug!(
+                        self.logger,
+                        "Could not find description for {}", &listing_url
+                    );
+
+                    None
+                }
+                Some(description) => {
+                    debug!(self.logger, "Found description for {}", &listing_url);
+
+                    Some(description.text().collect::<String>().trim().to_string())
+                }
+            };
 
             let mut id = html
                 .select(&self.id_selector)