Initial commit for Amazon Web Scraper python script and prerequisites

Scripts/Web_Scrappers/Amazon_products_scraper/AmazonScraper.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+from selectorlib import Extractor
+import requests 
+import json
+
+
+# CSS selectors of the product we will scrape.
+e = Extractor.from_yaml_file('selectors.yml')
+
+def scrape(url):  
+
+    # The headers our scraper will use.
+    headers = {
+        'dnt': '1',
+        'upgrade-insecure-requests': '1',
+        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36',
+        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        'sec-fetch-site': 'same-origin',
+        'sec-fetch-mode': 'navigate',
+        'sec-fetch-user': '?1',
+        'sec-fetch-dest': 'document',
+        'referer': 'https://www.amazon.com/',
+        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
+    }
+
+    # Download page using requests lib.
+    print("Downloading %s"%url)
+    req = requests.get(url, headers=headers)
+    # This is just to check if Amazon blocked our scraper.
+    if req.status_code > 500:
+        if "Contact for access to amazon data." in req.text:
+            print("Amazon Page: %s was blocked by Amazon. Please try using better proxies\n"%url)
+        else:
+            print("Amazon Page: %s was blocked, status code is: %d"%(url,req.status_code))
+        return None
+    # Return scraped values as text.
+    return e.extract(req.text)
+
+with open("search_products.txt",'r') as urllist, open('product_search_output.jsonl','w') as outfile:
+    for url in urllist.read().splitlines():
+        out_data = scrape(url) 
+        if out_data:
+            for product in out_data['products']:    
+                print(product)
+                product['search_url'] = url
+                print("Saving Product: %s"%product['title'])
+                json.dump(product,outfile)
+                outfile.write("\n")

Scripts/Web_Scrappers/Amazon_products_scraper/README.md

@@ -0,0 +1,26 @@
+# Amazon Products Scraper 
+
+Scrapes all the data from a given amazon search result url.
+
+### Pre-requisites
+
+	- Python 3
+	- requests
+	- selectorlib
+
+### Instructions to run
+	Input your item search url in the search_products.txt
+		Ex:
+			https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2
+
+  	> python3 AmazonScraper.py
+
+  	Output will be in a jsonl format and can be found in the product_search_output.jsonl file.
+
+## Screenshot - Sample Use
+
+![Screenshot](output.PNG)
+
+## Author
+
+	Benji (https://github.com/CaptainRoberts)

Scripts/Web_Scrappers/Amazon_products_scraper/product_search_output.jsonl

@@ -0,0 +1,22 @@
+{"title": "HP VH240a 23.8-inch Full HD 1080p IPS LED Monitor with Built-in Speakers and VESA Mounting, Rotating Portrait & Landscape, Tilt, and HDMI & VGA Ports (1KL30AA) - Black", "url": "/HP-23-8-inch-Adjustment-Speakers-VH240a/dp/B072M34RQC/ref=sr_1_1?dchild=1&keywords=monitors&qid=1601701865&sr=8-1", "rating": "4.6 out of 5 stars", "reviews": "12,163", "price": null, "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "LG 24M47VQ 24-Inch LED-lit Monitor, Black", "url": "/LG-24M47VQ-24-Inch-LED-lit-Monitor/dp/B00W95RR32/ref=sr_1_2?dchild=1&keywords=monitors&qid=1601701865&sr=8-2", "rating": "4.6 out of 5 stars", "reviews": "1,069", "price": "$99.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Acer CB272 bmiprx 27\" Full HD (1920 x 1080) IPS Zero Frame Home Office Monitor with AMD Radeon Free Sync - 1ms VRB, 75Hz Refresh, Height Adjustable Stand with Tilt & Pivot (Display, HDMI & VGA ports)", "url": "/Acer-CB272-bmiprx-FreeSync-Technology/dp/B07WMTLW2R/ref=sr_1_3?dchild=1&keywords=monitors&qid=1601701865&sr=8-3", "rating": "4.4 out of 5 stars", "reviews": "1,527", "price": "$154.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Dell P2419H 24 Inch LED-Backlit, Anti-Glare, 3H Hard Coating IPS Monitor - (8 ms Response, FHD 1920 x 1080 at 60Hz, 1000:1 Contrast, with ComfortView DisplayPort, VGA, HDMI and USB), Black", "url": "/Dell-Screen-LED-Lit-Monitor-P2419H/dp/B07F8XZN69/ref=sr_1_4?dchild=1&keywords=monitors&qid=1601701865&sr=8-4", "rating": "4.7 out of 5 stars", "reviews": "2,118", "price": null, "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "LG 29WN600-W 29\" 21:9 UltraWide WFHD IPS HDR1 0 Monitor with FreeSync", "url": "/gp/slredirect/picassoRedirect.html/ref=pa_sp_mtf_aps_sr_pg1_1?ie=UTF8&adId=A008008011AOJX32XRZUM&url=%2FLG-29WN600-W-29-21-UltraWide%2Fdp%2FB0876DBCBX%2Fref%3Dsr_1_5_sspa%3Fdchild%3D1%26keywords%3Dmonitors%26qid%3D1601701865%26sr%3D8-5-spons%26psc%3D1&qualifier=1601701864&id=521815347020263&widgetName=sp_mtf", "rating": "4.6 out of 5 stars", "reviews": "573", "price": "$229.00", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Sceptre IPS 24\u201d Gaming Monitor 165Hz 144Hz Full HD (1920 x 1080) FreeSync Eye Care FPS RTS DisplayPort HDMI Build-in Speakers, Machine Black 2020 (E248B-FPT168)", "url": "/Sceptre-FreeSync-DisplayPort-Speakers-E248B-FPT168/dp/B088VMY23M/ref=sr_1_6?dchild=1&keywords=monitors&qid=1601701865&sr=8-6", "rating": "4.5 out of 5 stars", "reviews": "820", "price": "$179.97", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "HP Pavilion 22cwa 21.5-Inch Full HD 1080p IPS LED Monitor, Tilt, VGA and HDMI (T4Q59AA) - Black", "url": "/HP-Pavilion-21-5-Inch-Monitor-22cwa/dp/B015WCV70W/ref=sr_1_7?dchild=1&keywords=monitors&qid=1601701865&sr=8-7", "rating": "4.5 out of 5 stars", "reviews": "7,314", "price": null, "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Asus VP28UQG 28\" Monitor 4K/UHD 3840x2160 1ms DP HDMI Adaptive Sync/FreeSync Eye Care Monitor", "url": "/VP28UQG-Monitor-3840x2160-Adaptive-FreeSync/dp/B074PK4R2H/ref=sr_1_8?dchild=1&keywords=monitors&qid=1601701865&sr=8-8", "rating": "4.4 out of 5 stars", "reviews": "1,030", "price": "$249.84", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Sceptre 24\" Curved 75Hz Gaming LED Monitor Full HD 1080P HDMI VGA Speakers, VESA Wall Mount Ready Metal Black 2019 (C248W-1920RN)", "url": "/Sceptre-Curved-Monitor-Speakers-C248W-1920RN/dp/B07KXSR99Y/ref=sr_1_9?dchild=1&keywords=monitors&qid=1601701865&sr=8-9", "rating": "4.6 out of 5 stars", "reviews": "7,032", "price": "$129.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "LG 34WN650-W 34-Inch 21:9 UltraWide Full HD (2560 x 1080) IPS Display with VESA DisplayHDR 400 and AMD FreeSync, Silver", "url": "/gp/slredirect/picassoRedirect.html/ref=pa_sp_mtf_aps_sr_pg1_1?ie=UTF8&adId=A0686664MM8CVA6DPJ15&url=%2FLG-34WN650-W-34-Inch-UltraWide-DisplayHDR%2Fdp%2FB087JB656Q%2Fref%3Dsr_1_10_sspa%3Fdchild%3D1%26keywords%3Dmonitors%26qid%3D1601701865%26sr%3D8-10-spons%26psc%3D1&qualifier=1601701864&id=521815347020263&widgetName=sp_mtf", "rating": "4.3 out of 5 stars", "reviews": "17", "price": "$398.31", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Acer SB220Q bi 21.5 Inches Full HD (1920 x 1080) IPS Ultra-Thin Zero Frame Monitor (HDMI & VGA port),Black", "url": "/Acer-SB220Q-Ultra-Thin-Frame-Monitor/dp/B07CVL2D2S/ref=sr_1_11?dchild=1&keywords=monitors&qid=1601701865&sr=8-11", "rating": "4.6 out of 5 stars", "reviews": "14,564", "price": null, "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Sceptre E275W-19203R 27\" Ultra Thin 1080P LED Monitor 2X HDMI VGA Build-In Speakers, Metallic Black 2018", "url": "/Sceptre-E275W-19203R-Monitor-Speakers-Metallic/dp/B077SRKKHQ/ref=sr_1_12?dchild=1&keywords=monitors&qid=1601701865&sr=8-12", "rating": "4.5 out of 5 stars", "reviews": "1,346", "price": "$109.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Dell 27 LED backlit LCD Monitor SE2719H IPS Full HD 1080p 1920 x 1080 at 60 Hz HDMI VGA,Black", "url": "/Dell-backlit-Monitor-SE2719H-1080p/dp/B07KW6HFD1/ref=sr_1_13?dchild=1&keywords=monitors&qid=1601701865&sr=8-13", "rating": "4.7 out of 5 stars", "reviews": "1,517", "price": null, "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "AOC CQ27G2 27\" Super Curved Frameless Gaming Monitor QHD 2K, 1500R Curved VA, 1ms, 144Hz, FreeSync, Height adjustable, 3-Year Zero Dead Pixel Guarantee", "url": "/AOC-CQ27G2-Frameless-Adjustable-Guarantee/dp/B0862YHJ56/ref=sr_1_14?dchild=1&keywords=monitors&qid=1601701865&sr=8-14", "rating": "4.6 out of 5 stars", "reviews": "8,614", "price": "$259.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "MOUNTUP Single Monitor Desk Mount - Adjustable Gas Spring Monitor Arm, VESA Mount with C Clamp, Grommet Mounting Base, Computer Monitor Stand for Screen up to 32 inch, MU0004", "url": "/gp/slredirect/picassoRedirect.html/ref=pa_sp_mtf_aps_sr_pg1_1?ie=UTF8&adId=A05196101JG2LCGN31JCL&url=%2FMOUNTUP-Single-Monitor-Desk-Mount%2Fdp%2FB07Z7ZGHD5%2Fref%3Dsr_1_15_sspa%3Fdchild%3D1%26keywords%3Dmonitors%26qid%3D1601701865%26sr%3D8-15-spons%26psc%3D1&qualifier=1601701864&id=521815347020263&widgetName=sp_mtf", "rating": "4.4 out of 5 stars", "reviews": "1,395", "price": "$27.90", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Dell SE2419Hx 23.8\" IPS Full HD (1920x1080) Monitor, Black", "url": "/Dell-SE2419Hx-23-8-1920x1080-Monitor/dp/B07HKV5RLG/ref=sr_1_16?dchild=1&keywords=monitors&qid=1601701865&sr=8-16", "rating": "4.7 out of 5 stars", "reviews": "1,205", "price": null, "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "ViewSonic VX3276-MHD 32 Inch 1080p Frameless Widescreen IPS Monitor with Screen Split Capability HDMI and DisplayPort", "url": "/ViewSonic-VX3276-MHD-Frameless-Widescreen-DisplayPort/dp/B0787WGCXT/ref=sr_1_17?dchild=1&keywords=monitors&qid=1601701865&sr=8-17", "rating": "4.6 out of 5 stars", "reviews": "2,309", "price": "$199.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "ViewSonic VS2210-H 22 inch Frameless 1080P IPS Monitor with Mega Dynamic Contrast ratio, Blue Light Filter, and HDMI, Black", "url": "/ViewSonic-VS2210-H-Frameless-Monitor-Contrast/dp/B07W6J5K8L/ref=sr_1_18?dchild=1&keywords=monitors&qid=1601701865&sr=8-18", "rating": "4.6 out of 5 stars", "reviews": "2,324", "price": "$89.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Acer R240HY bidx 23.8-Inch IPS HDMI DVI VGA (1920 x 1080) Widescreen Monitor, Black", "url": "/Acer-R240HY-bidx-23-8-Inch-Widescreen/dp/B0148NNKTC/ref=sr_1_19?dchild=1&keywords=monitors&qid=1601701865&sr=8-19", "rating": "4.6 out of 5 stars", "reviews": "7,015", "price": "$129.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "12 Inch Portable Monitor HDMI Computer Monitor HD 1366x768 TFT LCD Display with HDMI VGA BNC AV MIC Input Audio Video for PS4 Raspberry pi Mini PC CCTV DVR Home Office Surveillance, 75x75mm VESA", "url": "/gp/slredirect/picassoRedirect.html/ref=pa_sp_btf_aps_sr_pg1_1?ie=UTF8&adId=A01011642J9XGVEQWP37J&url=%2FPortable-Computer-1366x768-Raspberry-Surveillance%2Fdp%2FB0888JYY53%2Fref%3Dsr_1_20_sspa%3Fdchild%3D1%26keywords%3Dmonitors%26qid%3D1601701865%26sr%3D8-20-spons%26psc%3D1&qualifier=1601701864&id=521815347020263&widgetName=sp_btf", "rating": "4.2 out of 5 stars", "reviews": "7", "price": "$88.99", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "Philips 271B8QJEB 27\" monitor, Full HD 1920x1080 IPS, HDMI/DisplayPort/DVI-D/VGA, speakers, USB 3.0 hub, SoftBlue, Flicker-Free, Ergonomic base, VESA, TCO Edge", "url": "/gp/slredirect/picassoRedirect.html/ref=pa_sp_btf_aps_sr_pg1_1?ie=UTF8&adId=A0570155VJASKXIPVQDC&url=%2FPhilips-271B8QJEB-Class-Monitor-Adaptive%2Fdp%2FB07CQTL7LB%2Fref%3Dsr_1_21_sspa%3Fdchild%3D1%26keywords%3Dmonitors%26qid%3D1601701865%26sr%3D8-21-spons%26psc%3D1&qualifier=1601701864&id=521815347020263&widgetName=sp_btf", "rating": "5.0 out of 5 stars", "reviews": "5", "price": "$229.00", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}
+{"title": "AOC 27B1H 27' Full HD 1920x1080 Monitor, 3-Sided Frameless, IPS Panel, HDMI/VGA, Flicker-Free (Renewed)", "url": "/gp/slredirect/picassoRedirect.html/ref=pa_sp_btf_aps_sr_pg1_1?ie=UTF8&adId=A0562609TEIJ4B56DFC&url=%2FAOC-1920x1080-Monitor-Frameless-Flicker-Free%2Fdp%2FB07P1ZSR38%2Fref%3Dsr_1_22_sspa%3Fdchild%3D1%26keywords%3Dmonitors%26qid%3D1601701865%26sr%3D8-22-spons%26psc%3D1&qualifier=1601701864&id=521815347020263&widgetName=sp_btf", "rating": "4.0 out of 5 stars", "reviews": "66", "price": "$159.00", "search_url": "https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2"}

Scripts/Web_Scrappers/Amazon_products_scraper/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.24.0
+selectorlib==0.16.0

Scripts/Web_Scrappers/Amazon_products_scraper/search_products.txt

@@ -0,0 +1 @@
+https://www.amazon.com/s?k=monitors&ref=nb_sb_noss_2

Scripts/Web_Scrappers/Amazon_products_scraper/selectors.yml

@@ -0,0 +1,28 @@
+products:
+    css: 'div[data-component-type="s-search-result"]'
+    xpath: null
+    multiple: true
+    type: Text
+    children:
+        title:
+            css: 'h2 a.a-link-normal.a-text-normal'
+            xpath: null
+            type: Text
+        url:
+            css: 'h2 a.a-link-normal.a-text-normal'
+            xpath: null
+            type: Link
+        rating:
+            css: 'div.a-row.a-size-small span:nth-of-type(1)'
+            xpath: null
+            type: Attribute
+            attribute: aria-label
+        reviews:
+            css: 'div.a-row.a-size-small span:nth-of-type(2)'
+            xpath: null
+            type: Attribute
+            attribute: aria-label
+        price:
+            css: 'span.a-price:nth-of-type(1) span.a-offscreen'
+            xpath: null
+            type: Text