add admin dep

api.py

@@ -97,7 +97,11 @@ def data_fetch_loop():
     for webpage in webpages:
         print("Processing",webpage.id)
         try:
-            price = next(filter(lambda ext: ext.is_valid_url(webpage.url),extractors)).extract_data(webpage.url)
+            extractor = next(filter(lambda ext: ext.is_valid_url(webpage.url),extractors))
+            if extractor is None:
+                print("No extractor found for", webpage.url)
+                continue
+            price = extractor.extract_data(webpage.url)
             obj = {
                 "webpage_id": webpage.id,
                 "url": webpage.url,
@@ -114,7 +118,7 @@ def data_fetch_loop():
 
     is_fetching = False
 
-job = scheduler.add_job(data_fetch_loop, 'interval', seconds=180)
+job = scheduler.add_job(data_fetch_loop, 'interval', seconds=60*5)
 
 scheduler.start()
 
@@ -131,7 +135,7 @@ def wrapper(*args, **kwargs):
             else:
                 return jsonify({"error": "Not authorized"}), 401
         else:
-            func(*args, **kwargs)
+            return func(*args, **kwargs)
     return wrapper
 
 @api.before_request
@@ -145,6 +149,12 @@ def syncThreads():
         db_session.commit()
 
 # when your browser wants the data for the the site item
+
+@api.route('/force_refetch')
+def force_resync():
+    data_fetch_loop()
+    return "Gotcha!"
+
 @api.route('/all_webpages')
 def everything():
     return jsonify(db_session.query(Webpage).all())

extractors/BestBuyExtractor.py

@@ -0,0 +1,18 @@
+from extractors.ExtractorBase import ExtractorBase
+
+from bs4 import BeautifulSoup
+
+class BestBuyExtractor(ExtractorBase):
+    def __init__(self):
+        super(BestBuyExtractor, self).__init__()
+    def extract_data(self,url):
+        ExtractorBase.extract_data(self,url) # superclass call
+        content = self.driver.page_source
+        soup = BeautifulSoup(content, 'lxml')
+        priceElem = next(soup.find('div', {
+                'class': ['priceView-hero-price','priceView-customer-price']
+        }).children)
+        return float(priceElem.getText().replace("$",""))
+
+    def is_valid_url(self, url: str):
+        return url.startswith('https://www.bestbuy.com') or url.startswith('https://bestbuy.com')

extractors/ExtractorBase.py

@@ -4,12 +4,36 @@
 
 import os
 class ExtractorBase:
+    globalDriver = None
+
     def __init__(self):
+        if ExtractorBase.globalDriver is not None:
+            self.driver = ExtractorBase.globalDriver
+            return
         if os.getenv('PHANTOMJS_LOCATION') is None:
-            self.driver = webdriver.Chrome(os.getenv('CHROMEDRIVER_LOCATION','/home/raymond/chromedriver'))
+            options = webdriver.ChromeOptions()
+            # options.add_argument("start-maximized")
+            options.add_experimental_option("excludeSwitches", ["enable-automation"])
+            options.add_experimental_option('useAutomationExtension', False)
+            self.driver = webdriver.Chrome(os.getenv('CHROMEDRIVER_LOCATION','/home/raymond/chromedriver'), options = options)
+            try:
+                from selenium_stealth import stealth
+                stealth(self.driver,
+                    languages=["en-US", "en"],
+                    vendor="Google Inc.",
+                    platform="Win32",
+                    webgl_vendor="Google Inc. (NVIDIA Corporation)",
+                    renderer="ANGLE (NVIDIA Corporation, NVIDIA GeForce GTX 1650/PCIe/SSE2, OpenGL 4.5.0 NVIDIA 470.74)",
+                    fix_hairline=True
+                )
+            except:
+                print("Selenium Stealth not found, this may increase your changes of being detected as a bot")
+
         else:
-            self.driver = webdriver.PhantomJS(os.getenv('PHANTOMJS_LOCATION'))
+            self.driver = webdriver.PhantomJS(executable_path=os.getenv('PHANTOMJS_LOCATION'))
         self.driver.set_window_size(1920, 960)
+        # self.driver.set_page_load_timeout(60)
+        ExtractorBase.globalDriver = self.driver
     def extract_data(self, url):
         # return the relevant data for a url
         self.driver.get(url)

extractors/Multiextractor.py

@@ -8,7 +8,7 @@ def __init__(self):
     def extract_data(self,url):
         ExtractorBase.extract_data(self,url) # superclass call
         content = self.driver.page_source
-        soup = BeautifulSoup(content, 'html.parser')
+        soup = BeautifulSoup(content, 'lxml')
         if "amazon" in url:                
             #for amazon
             # pw = soup.find('span', 'a-price-whole')
@@ -19,7 +19,7 @@ def extract_data(self,url):
             }).children)
             #returns price
             print(priceElem.getText())
-            return priceElem.getText().replace("$","")#pw.get_text() + pf.get_text()
+            return float(priceElem.getText().replace("$",""))#pw.get_text() + pf.get_text()
 
         elif "target" in url:
             #for Target
@@ -28,7 +28,7 @@ def extract_data(self,url):
             })
             #returns price only
             print(pw,pw.getText())
-            return pw.getText().replace("$","")
+            return float(pw.getText().replace("$",""))
         elif "google" in url:
             #for google shopping
             # div class for airpod, mac,  brands -> aULzUe IuHnof

extractors/WalmartExtractor.py

@@ -0,0 +1,18 @@
+from extractors.ExtractorBase import ExtractorBase
+
+from bs4 import BeautifulSoup
+
+class WalmartExtractor(ExtractorBase):
+    def __init__(self):
+        super(WalmartExtractor, self).__init__()
+    def extract_data(self,url):
+        ExtractorBase.extract_data(self,url) # superclass call
+        content = self.driver.page_source
+        soup = BeautifulSoup(content, 'lxml')
+        priceElem = soup.find('div', {
+                'itemprop':  "price"
+        })
+        return float(priceElem.getText().replace("$",""))
+
+    def is_valid_url(self, url: str):
+        return url.startswith('https://www.walmart.com') or url.startswith('https://walmart.com')

extractors/all_extractors.py

@@ -1,5 +1,7 @@
 from .Multiextractor import MultiExtractor
-extractors = [MultiExtractor]
+from .BestBuyExtractor import BestBuyExtractor
+from .WalmartExtractor import WalmartExtractor
+extractors = [MultiExtractor,BestBuyExtractor,WalmartExtractor]
 
 def create_instances():
     instances = []

requirements.txt

@@ -10,4 +10,8 @@ Flask-Login
 python-dotenv
 snowflake-id
 selenium
-apscheduler
+apscheduler
+flask_admin
+BeautifulSoup4
+cchardet
+lxml

static/js/all_products.js

@@ -1,7 +1,7 @@
 function migrateIfNeeded(data) {
     if(!data.price){
         data.price = "-1";
-    }else if(data.price.includes("$")){
+    }else if(typeof data.price === "string" && data.price.includes("$")){
         data.price = data.price.replace("$", "");
     }
     if(typeof data.price === "string"){
@@ -66,9 +66,14 @@ docReady(() => {
                     prodLink.href = webpageInfo.url;
                     prodLink.innerText = webpageInfo.url;
                     dataDiv.appendChild(prodLink);
-
+                    
                     let priceDiv = document.createElement("div");
                     priceDiv.className = "product-last-price";
+                    if(collectedWebpageData.length == 0){
+                        priceDiv.innerText = "No data collected yet. ";
+                        dataDiv.appendChild(priceDiv);
+                        return;
+                    }
                     priceDiv.innerText = "Last price: " + collectedWebpageData[0].price;
                     dataDiv.appendChild(priceDiv);
 
@@ -77,7 +82,7 @@ docReady(() => {
                     priceGraph.id = "price-graph-" + product.id + "-" + webpage.id;
                     priceGraph.width = "800";
                     priceGraph.height = "600";
-
+                    
                     // maxmin
                     let arrMax = collectedWebpageData.map(s => s.price).reduce((a,b) => Math.max(a,b));
                     let arrMin = collectedWebpageData.map(s => s.price).reduce((a,b) => Math.min(a,b));
@@ -137,9 +142,11 @@ docReady(() => {
                 dataDiv.className = "product-all-data";
             }));
         }catch(ex){
+            console.log(ex);
             set_alert_danger("Error" + ex);
         }
     }).catch(err => {
+        console.log(err);
         set_alert_danger("Error" + err);
     });
 });

templates/layouts/main.html

@@ -48,6 +48,9 @@
           <li class="nav-item active">
             <a class="nav-link" href="/">Home</a>
           </li>
+          <li class="nav-item">
+            <a class="nav-link" href={{ url_for('products') }}>Product Listing</a>
+          </li>
           <li class="nav-item">
             <a class="nav-link" href={{ url_for('create_product') }}>Create Product</a>
           </li>
@@ -57,6 +60,7 @@
             </a>
             <div class="dropdown-menu" aria-labelledby="navbarDropdown">
               <a class="dropdown-item" href={{ url_for('about' )}}>About</a>
+              <a class="dropdown-item" href={{ url_for('add_webpage' )}}>Add Webpage</a>
             </div>
           </li>
         </ul>

templates/pages/products.html

@@ -1,5 +1,5 @@
 {% extends 'layouts/main.html' %}
-{% block title %}Add Webpage{% endblock %}
+{% block title %}Product Status{% endblock %}
 {% block content %}
 <script src="https://cdn.jsdelivr.net/npm/chart.js@3.6.0/dist/chart.min.js" integrity="sha256-7lWo7cjrrponRJcS6bc8isfsPDwSKoaYfGIHgSheQkk=" crossorigin="anonymous"></script>
 <script src="/static/js/all_products.js"></script>