Codes/WebScraping_scheduler.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "auto_scraper.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMmCI/SUyFKe/7qz9OiBwHQ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DSPG-2022/Local-Food-Benchmarking/blob/main/auto_scraper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load Libraries"
      ],
      "metadata": {
        "id": "b5iSCBYzzetl"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7QsMZeY2D28h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a5c9c997-7a58-4c02-f77a-34f83ee5b1d8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.11) or chardet (3.0.4) doesn't match a supported version!\n",
            "  RequestsDependencyWarning)\n",
            "/usr/local/lib/python3.7/dist-packages/trio/_core/_multierror.py:464: RuntimeWarning: IPython detected, but you already have a custom exception handler installed. I'll skip installing Trio's custom handler, but this means MultiErrors will not show full tracebacks.\n",
            "  category=RuntimeWarning,\n"
          ]
        }
      ],
      "source": [
        "# !pip install chompjs\n",
        "# !pip install requests-html\n",
        "\n",
        "import json\n",
        "import csv\n",
        "import re\n",
        "import time\n",
        "import pandas as pd\n",
        "import requests\n",
        "from requests_html import HTMLSession\n",
        "from datetime import date\n",
        "from csv import DictWriter"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Follow intructions in [userDoc_autoscraper.pdf](https://github.com/DSPG-2022/Local-Food-Benchmarking/blob/main/notebooks/userDoc_autoscraper.pdf) to use [Postman API](https://www.postman.com/) to retreive the API response from the website."
      ],
      "metadata": {
        "id": "hKcFrsvLz019"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Create an empty list to store results\n",
        "\n",
        "result = []\n",
        "\n",
        "#Loop through url pages to get product information\n",
        "\n",
        "for x in range(1,6): \n",
        "  url = f\"https://www.walmart.com/orchestra/home/graphql/browse?affinityOverride=default&page={x}&prg=desktop&catId=976759_976793_9756351&sort=best_match&ps=40&searchArgs.cat_id=976759_976793_9756351&searchArgs.prg=desktop&fitmentFieldParams=true&fetchMarquee=true&fetchSkyline=true&fetchSbaTop=false&enablePortableFacets=true&tenant=WM_GLASS&enableFacetCount=true\"\n",
        "  payload = json.dumps({\n",
        "    \"query\": \"query Browse( $query:String $page:Int $prg:Prg! $facet:String $sort:Sort $catId:String! $max_price:String $min_price:String $module_search:String $affinityOverride:AffinityOverride $ps:Int $ptss:String $beShelfId:String $fitmentFieldParams:JSON ={}$fitmentSearchParams:JSON ={}$rawFacet:String $seoPath:String $trsp:String $fetchMarquee:Boolean! $fetchSkyline:Boolean! $additionalQueryParams:JSON ={}$enablePortableFacets:Boolean = false $intentSource:IntentSource $tenant:String! $enableFacetCount:Boolean = true $pageType:String! = \\\"BrowsePage\\\" ){search( query:$query page:$page prg:$prg facet:$facet sort:$sort cat_id:$catId max_price:$max_price min_price:$min_price module_search:$module_search affinityOverride:$affinityOverride additionalQueryParams:$additionalQueryParams ps:$ps ptss:$ptss trsp:$trsp intentSource:$intentSource _be_shelf_id:$beShelfId pageType:$pageType ){query searchResult{...BrowseResultFragment}}contentLayout( channel:\\\"WWW\\\" pageType:$pageType tenant:$tenant version:\\\"v1\\\" searchArgs:{query:$query cat_id:$catId _be_shelf_id:$beShelfId prg:$prg}){modules{...ModuleFragment configs{...on EnricherModuleConfigsV1{zoneV1}__typename...on _TempoWM_GLASSWWWSearchSortFilterModuleConfigs{facetsV1 @skip(if:$enablePortableFacets){...FacetFragment}topNavFacets @include(if:$enablePortableFacets){...FacetFragment}allSortAndFilterFacets @include(if:$enablePortableFacets){...FacetFragment}}...on TempoWM_GLASSWWWPillsModuleConfigs{moduleSource pillsV2{...PillsModuleFragment}}...TileTakeOverProductFragment...on TempoWM_GLASSWWWSearchFitmentModuleConfigs{fitments( fitmentSearchParams:$fitmentSearchParams fitmentFieldParams:$fitmentFieldParams ){...FitmentFragment sisFitmentResponse{...BrowseResultFragment}}}...on TempoWM_GLASSWWWStoreSelectionHeaderConfigs{fulfillmentMethodLabel storeDislayName}...on TempoWM_GLASSWWWSponsoredProductCarouselConfigs{_rawConfigs}...PopularInModuleFragment...CopyBlockModuleFragment...BannerModuleFragment...HeroPOVModuleFragment...InlineSearchModuleFragment...MarqueeDisplayAdConfigsFragment @include(if:$fetchMarquee)...SkylineDisplayAdConfigsFragment @include(if:$fetchSkyline)...HorizontalChipModuleConfigsFragment...SkinnyBannerFragment}}...LayoutFragment pageMetadata{location{pickupStore deliveryStore intent postalCode stateOrProvinceCode city storeId accessPointId accessType spokeNodeId}pageContext}}seoBrowseMetaData( id:$catId facets:$rawFacet path:$seoPath facet_query_param:$facet _be_shelf_id:$beShelfId ){metaTitle metaDesc metaCanon h1 noIndex}}fragment BrowseResultFragment on SearchInterface{title aggregatedCount...BreadCrumbFragment...ShelfDataFragment...DebugFragment...ItemStacksFragment...PageMetaDataFragment...PaginationFragment...RequestContextFragment...ErrorResponse modules{facetsV1 @skip(if:$enablePortableFacets){...FacetFragment}topNavFacets @include(if:$enablePortableFacets){...FacetFragment}allSortAndFilterFacets @include(if:$enablePortableFacets){...FacetFragment}pills{...PillsModuleFragment}}}fragment ModuleFragment on TempoModule{name version type moduleId schedule{priority}matchedTrigger{zone}}fragment LayoutFragment on ContentLayout{layouts{id layout}}fragment BreadCrumbFragment on SearchInterface{breadCrumb{id name url}}fragment ShelfDataFragment on SearchInterface{shelfData{shelfName shelfId}}fragment DebugFragment on SearchInterface{debug{sisUrl adsUrl}}fragment ItemStacksFragment on SearchInterface{itemStacks{displayMessage meta{adsBeacon{adUuid moduleInfo max_ads}query stackId stackType title layoutEnum totalItemCount totalItemCountDisplay viewAllParams{query cat_id sort facet affinityOverride recall_set min_price max_price}}itemsV2{...ItemFragment...InGridMarqueeAdFragment...TileTakeOverTileFragment}}}fragment ItemFragment on Product{__typename id usItemId fitmentLabel name checkStoreAvailabilityATC seeShippingEligibility brand type shortDescription imageInfo{...ProductImageInfoFragment}canonicalUrl externalInfo{url}itemType category{path{name url}}badges{flags{...on BaseBadge{key text type id}...on PreviouslyPurchasedBadge{id text key lastBoughtOn numBought}}tags{...on BaseBadge{key text type}}}classType averageRating numberOfReviews esrb mediaRating salesUnitType sellerId sellerName hasSellerBadge availabilityStatusV2{display value}groupMetaData{groupType groupSubType numberOfComponents groupComponents{quantity offerId componentType productDisplayName}}productLocation{displayValue aisle{zone aisle}}fulfillmentSpeed offerId preOrder{...PreorderFragment}priceInfo{...ProductPriceInfoFragment}variantCriteria{...VariantCriteriaFragment}snapEligible fulfillmentBadge fulfillmentTitle fulfillmentType brand manufacturerName showAtc sponsoredProduct{spQs clickBeacon spTags viewBeacon}showOptions showBuyNow rewards{eligible state minQuantity rewardAmt promotionId selectionToken cbOffer term expiry description}}fragment ProductImageInfoFragment on ProductImageInfo{thumbnailUrl size}fragment ProductPriceInfoFragment on ProductPriceInfo{priceRange{minPrice maxPrice}currentPrice{...ProductPriceFragment}wasPrice{...ProductPriceFragment}unitPrice{...ProductPriceFragment}listPrice{...ProductPriceFragment}shipPrice{...ProductPriceFragment}subscriptionPrice{priceString subscriptionString}priceDisplayCodes{priceDisplayCondition finalCostByWeight submapType}}fragment PreorderFragment on PreOrder{isPreOrder preOrderMessage preOrderStreetDateMessage}fragment ProductPriceFragment on ProductPrice{price priceString}fragment VariantCriteriaFragment on VariantCriterion{name type id isVariantTypeSwatch variantList{id images name rank swatchImageUrl availabilityStatus products selectedProduct{canonicalUrl usItemId}}}fragment InGridMarqueeAdFragment on MarqueePlaceholder{__typename type moduleLocation lazy}fragment TileTakeOverTileFragment on TileTakeOverProductPlaceholder{__typename type tileTakeOverTile{span title subtitle image{src alt}logoImage{src alt}backgroundColor titleTextColor subtitleTextColor tileCta{ctaLink{clickThrough{value}linkText title}ctaType ctaTextColor}}}fragment PageMetaDataFragment on SearchInterface{pageMetadata{storeSelectionHeader{fulfillmentMethodLabel storeDislayName}title canonical description location{addressId}}}fragment PaginationFragment on SearchInterface{paginationV2{maxPage pageProperties}}fragment RequestContextFragment on SearchInterface{requestContext{vertical isFitmentFilterQueryApplied searchMatchType categories{id name}}}fragment ErrorResponse on SearchInterface{errorResponse{correlationId source errorCodes errors{errorType statusCode statusMsg source}}}fragment PillsModuleFragment on PillsSearchInterface{title url image:imageV1{src alt}}fragment BannerModuleFragment on TempoWM_GLASSWWWSearchBannerConfigs{moduleType viewConfig{title image imageAlt displayName description url urlAlt appStoreLink appStoreLinkAlt playStoreLink playStoreLinkAlt}}fragment PopularInModuleFragment on TempoWM_GLASSWWWPopularInBrowseConfigs{seoBrowseRelmData(id:$catId){relm{id name url}}}fragment CopyBlockModuleFragment on TempoWM_GLASSWWWCopyBlockConfigs{copyBlock(id:$catId){cwc}}fragment FacetFragment on Facet{title name type layout min max selectedMin selectedMax unboundedMax stepSize isSelected values{id title name description type itemCount @include(if:$enableFacetCount) isSelected baseSeoURL}}fragment FitmentFragment on Fitments{partTypeIDs result{status formId position quantityTitle extendedAttributes{...FitmentFieldFragment}labels{...LabelFragment}resultSubTitle notes suggestions{...FitmentSuggestionFragment}}labels{...LabelFragment}savedVehicle{vehicleType{...VehicleFieldFragment}vehicleYear{...VehicleFieldFragment}vehicleMake{...VehicleFieldFragment}vehicleModel{...VehicleFieldFragment}additionalAttributes{...VehicleFieldFragment}}fitmentFields{...VehicleFieldFragment}fitmentForms{id fields{...FitmentFieldFragment}title labels{...LabelFragment}}}fragment LabelFragment on FitmentLabels{ctas{...FitmentLabelEntityFragment}messages{...FitmentLabelEntityFragment}links{...FitmentLabelEntityFragment}images{...FitmentLabelEntityFragment}}fragment FitmentLabelEntityFragment on FitmentLabelEntity{id label}fragment VehicleFieldFragment on FitmentVehicleField{id label value}fragment FitmentFieldFragment on FitmentField{id displayName value extended data{value label}dependsOn}fragment FitmentSuggestionFragment on FitmentSuggestion{id position loadIndex speedRating searchQueryParam labels{...LabelFragment}cat_id fitmentSuggestionParams{id value}}fragment HeroPOVModuleFragment on TempoWM_GLASSWWWHeroPovConfigsV1{povCards{card{povStyle image{mobileImage{...TempoCommonImageFragment}desktopImage{...TempoCommonImageFragment}}heading{text textColor textSize}subheading{text textColor}detailsView{backgroundColor isTransparent}ctaButton{button{linkText clickThrough{value}uid}}legalDisclosure{regularText shortenedText textColor textColorMobile legalBottomSheetTitle legalBottomSheetDescription}logo{...TempoCommonImageFragment}links{link{linkText}}}}}fragment TempoCommonImageFragment on TempoCommonImage{src alt assetId uid clickThrough{value}}fragment InlineSearchModuleFragment on TempoWM_GLASSWWWInlineSearchConfigs{headingText placeholderText}fragment MarqueeDisplayAdConfigsFragment on TempoWM_GLASSWWWMarqueeDisplayAdConfigs{_rawConfigs ad{...DisplayAdFragment}}fragment DisplayAdFragment on Ad{...AdFragment adContent{type data{__typename...AdDataDisplayAdFragment}}}fragment AdFragment on Ad{status moduleType platform pageId pageType storeId stateCode zipCode pageContext moduleConfigs adsContext adRequestComposite}fragment AdDataDisplayAdFragment on AdData{...on DisplayAd{json status}}fragment SkylineDisplayAdConfigsFragment on TempoWM_GLASSWWWSkylineDisplayAdConfigs{_rawConfigs ad{...SkylineDisplayAdFragment}}fragment SkylineDisplayAdFragment on Ad{...SkylineAdFragment adContent{type data{__typename...SkylineAdDataDisplayAdFragment}}}fragment SkylineAdFragment on Ad{status moduleType platform pageId pageType storeId stateCode zipCode pageContext moduleConfigs adsContext adRequestComposite}fragment SkylineAdDataDisplayAdFragment on AdData{...on DisplayAd{json status}}fragment HorizontalChipModuleConfigsFragment on TempoWM_GLASSWWWHorizontalChipModuleConfigs{chipModuleSource:moduleSource chipModule{title url{linkText title clickThrough{type value}}}chipModuleWithImages{title url{linkText title clickThrough{type value}}image{alt clickThrough{type value}height src title width}}}fragment SkinnyBannerFragment on TempoWM_GLASSWWWSkinnyBannerConfigs{bannerType desktopBannerHeight bannerImage{src title alt}mobileBannerHeight mobileImage{src title alt}clickThroughUrl{clickThrough{value}}backgroundColor heading{title fontColor}subHeading{title fontColor}bannerCta{ctaLink{linkText clickThrough{value}}textColor ctaType}}fragment TileTakeOverProductFragment on TempoWM_GLASSWWWTileTakeOverProductConfigs{dwebSlots mwebSlots TileTakeOverProductDetails{pageNumber span dwebPosition mwebPosition title subtitle image{src alt}logoImage{src alt}backgroundColor titleTextColor subtitleTextColor tileCta{ctaLink{clickThrough{value}linkText title}ctaType ctaTextColor}}}\",\n",
        "    \"variables\": {\n",
        "      \"id\": \"\",\n",
        "      \"affinityOverride\": \"default\",\n",
        "      \"dealsId\": \"\",\n",
        "      \"query\": \"\",\n",
        "      \"page\": x,\n",
        "      \"prg\": \"desktop\",\n",
        "      \"catId\": \"976759_976793_9756351\",\n",
        "      \"facet\": \"\",\n",
        "      \"sort\": \"best_match\",\n",
        "      \"rawFacet\": \"\",\n",
        "      \"seoPath\": \"\",\n",
        "      \"ps\": 40,\n",
        "      \"ptss\": \"\",\n",
        "      \"trsp\": \"\",\n",
        "      \"beShelfId\": \"\",\n",
        "      \"recall_set\": \"\",\n",
        "      \"module_search\": \"\",\n",
        "      \"min_price\": \"\",\n",
        "      \"max_price\": \"\",\n",
        "      \"storeSlotBooked\": \"\",\n",
        "      \"additionalQueryParams\": {\n",
        "        \"hidden_facet\": None,\n",
        "        \"translation\": None\n",
        "      },\n",
        "      \"searchArgs\": {\n",
        "        \"query\": \"\",\n",
        "        \"cat_id\": \"976759_976793_9756351\",\n",
        "        \"prg\": \"desktop\",\n",
        "        \"facet\": \"\"\n",
        "      },\n",
        "      \"fitmentFieldParams\": {\n",
        "        \"powerSportEnabled\": True\n",
        "      },\n",
        "      \"fitmentSearchParams\": {\n",
        "        \"id\": \"\",\n",
        "        \"affinityOverride\": \"default\",\n",
        "        \"dealsId\": \"\",\n",
        "        \"query\": \"\",\n",
        "        \"page\": x,\n",
        "        \"prg\": \"desktop\",\n",
        "        \"catId\": \"976759_976793_9756351\",\n",
        "        \"facet\": \"\",\n",
        "        \"sort\": \"best_match\",\n",
        "        \"rawFacet\": \"\",\n",
        "        \"seoPath\": \"\",\n",
        "        \"ps\": 40,\n",
        "        \"ptss\": \"\",\n",
        "        \"trsp\": \"\",\n",
        "        \"beShelfId\": \"\",\n",
        "        \"recall_set\": \"\",\n",
        "        \"module_search\": \"\",\n",
        "        \"min_price\": \"\",\n",
        "        \"max_price\": \"\",\n",
        "        \"storeSlotBooked\": \"\",\n",
        "        \"additionalQueryParams\": {\n",
        "          \"hidden_facet\": None,\n",
        "          \"translation\": None\n",
        "        },\n",
        "        \"searchArgs\": {\n",
        "          \"query\": \"\",\n",
        "          \"cat_id\": \"976759_976793_9756351\",\n",
        "          \"prg\": \"desktop\",\n",
        "          \"facet\": \"\"\n",
        "        },\n",
        "        \"cat_id\": \"976759_976793_9756351\",\n",
        "        \"_be_shelf_id\": \"\"\n",
        "      },\n",
        "      \"fetchMarquee\": True,\n",
        "      \"fetchSkyline\": True,\n",
        "      \"fetchSbaTop\": False,\n",
        "      \"enablePortableFacets\": True,\n",
        "      \"tenant\": \"WM_GLASS\",\n",
        "      \"enableFacetCount\": True,\n",
        "      \"pageType\": \"BrowsePage\"\n",
        "    }\n",
        "  })\n",
        "  headers = {\n",
        "    'authority': 'www.walmart.com',\n",
        "    'accept': 'application/json',\n",
        "    'accept-language': 'en-US,en;q=0.9',\n",
        "    'content-type': 'application/json',\n",
        "    'cookie': 'ACID=86dff2dd-8ce6-44a6-96eb-77c824bd47c9; hasACID=true; TB_Latency_Tracker_100=1; TB_Navigation_Preload_01=1; TB_SFOU-100=; vtc=TdSHZIomVyLFg458gx4NQk; _pxhd=9a0238df9653e0937e77109a1766a73c6f17d202a5ea38555bc257ee72d6ef4a:ea652c7f-0c25-11ed-8948-735870754a72; TS013ed49a=01538efd7c6c84a8bd2593b26d789638b90c07baf2c66119ebc733f3cdfba420d33333db4d80a3afcaddc168ca8a918e233f36b934; dimensionData=767; TBV=7; pxcts=eb8c2288-0c25-11ed-b0ae-65796e495169; _pxvid=ea652c7f-0c25-11ed-8948-735870754a72; locGuestData=eyJpbnRlbnQiOiJTSElQUElORyIsImlzRXhwbGljaXQiOmZhbHNlLCJzdG9yZUludGVudCI6IlBJQ0tVUCIsIm1lcmdlRmxhZyI6ZmFsc2UsImlzRGVmYXVsdGVkIjpmYWxzZSwicGlja3VwIjp7Im5vZGVJZCI6Ijc0OSIsInRpbWVzdGFtcCI6MTY1ODc1OTI1MjIxOH0sInBvc3RhbENvZGUiOnsidGltZXN0YW1wIjoxNjU4NzU5MjUyMjE4LCJiYXNlIjoiNTAwMTQifSwidmFsaWRhdGVLZXkiOiJwcm9kOnYyOjg2ZGZmMmRkLThjZTYtNDRhNi05NmViLTc3YzgyNGJkNDdjOSJ9; _astc=ddb03b20993178d277ee400209287e2e; tb_sw_supported=false; AID=wmlspartner%253D0%253Areflectorid%253D0000000000000000000000%253Alastupd%253D1658759270798; auth=MTAyOTYyMDE4uarsVtFe%2BDJ8ekqfpKdnt04bBjeOqT%2BhVYSSVS8g5mO2TpDZb0yXTNizHk%2BsCpUezLJ9IZr5wp2N6kLw88RbzfeAN3JRwF8LqzDRQDRtLEx6oak39V06%2BPYm4j%2FsjlrP767wuZloTfhm7Wk2Kcjygv3M5Jnvc7ePkiG6%2BkglNAB00G4lzM37PoXAtVS2BCtBAGgov0PvcVxW2rTeBwBywDZoxl%2Bs1F1Go3FSpLCFhjEUMk70P8glgOEpLOprhDfMM%2FFHGZ2dCNmxWrdkwqEKrj3ttqH2Vm5E85ixwLGPFoNFWK%2BJENsRJNyJxZdZerQi%2B3b03dvYV0JVHUbwaSeJS33Qd%2FlWbMY%2Boo6cEupT2G3OqzoojeuxCIda%2BEVmXR7hBAUmOJ%2F2f7%2FoKaVyYk6haEjyrOXbKKhH072NS%2FW0j%2FU%3D; assortmentStoreId=749; hasLocData=1; bstc=Sj1sMCsRTe4FYrsb-ZJ8zI; mobileweb=0; xpa=0wI6o|1A0pE|2Oqb6|2SWkj|55ZtN|927zv|Ao5F6|CN28l|HDfyl|IOIpg|Iui6D|K2DXb|LTD5Y|NPqGB|O1c3v|PzQ_l|Qd0ry|SmVSa|T-onc|W9IW0|Zemho|aPJIO|bcl64|ccDGr|duBe9|eEurn|elin2|iqBQY|lqVt_|oDpYF|ovdrI|qsDvB|qyn67|sbXp_|uHfV2|uru_L|zCylr; exp-ck=0wI6o12SWkj155ZtN1927zv1HDfyl2Iui6D1NPqGB1O1c3v2PzQ_l1Qd0ry1SmVSa1W9IW02Zemho1ccDGr1eEurn1iqBQY1ovdrI3qsDvB1qyn672; bm_mi=F924EC90442FEBA902C449F3C554E78E~YAAQVmDcFywCryOCAQAAdzpaNhDbUeTbkkZRsPV7aeXnUMMQp/A/i4NtFvRNF5H5VGsipK0Sao6dPXBb9dpFJRhXdSz4mXV+qR/8Th42Q0oebgm8j6biTzigJY1Jyf8fGhHCNsEw+E/yYimlX2tUMlZ7hp8EwG9BNDm+BNBZVh8MijnzXiHqm/9G2B3LRRnENmQDIPJz1XuRrc8fyq2+w8zEvJaswAYUzqB/SWaqr8RX/f+pOXHhhW7PdySAHZk/TaVXH5jRNAn3Jdqk5KO0AWyS7+QB4+7FQPFcPDQHNHWJ29HpLX0DAnhOndroD6UtwQYOjQihLuS5Sr7Us++IDyuh0jxJ7ndCgMrpDO+FlNmCvuu3WU1t8axyZL5A~1; xpm=1%2B1658769258%2BTdSHZIomVyLFg458gx4NQk~%2B0; locDataV3=eyJpc0RlZmF1bHRlZCI6ZmFsc2UsImlzRXhwbGljaXQiOmZhbHNlLCJpbnRlbnQiOiJTSElQUElORyIsInBpY2t1cCI6W3siYnVJZCI6IjAiLCJub2RlSWQiOiI3NDkiLCJkaXNwbGF5TmFtZSI6IkFtZXMgU3VwZXJjZW50ZXIiLCJub2RlVHlwZSI6IlNUT1JFIiwiYWRkcmVzcyI6eyJwb3N0YWxDb2RlIjoiNTAwMTAiLCJhZGRyZXNzTGluZTEiOiIzMTA1IEdyYW5kIEF2ZSIsImNpdHkiOiJBbWVzIiwic3RhdGUiOiJJQSIsImNvdW50cnkiOiJVUyIsInBvc3RhbENvZGU5IjoiNTAwMTAtNDIwMSJ9LCJnZW9Qb2ludCI6eyJsYXRpdHVkZSI6NDIuMDUzMzM3LCJsb25naXR1ZGUiOi05My42MjIwMzZ9LCJpc0dsYXNzRW5hYmxlZCI6dHJ1ZSwic2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZSwidW5TY2hlZHVsZWRFbmFibGVkIjp0cnVlLCJodWJOb2RlSWQiOiI3NDkiLCJzdG9yZUhycyI6IjA2OjAwLTIzOjAwIiwic3VwcG9ydGVkQWNjZXNzVHlwZXMiOlsiUElDS1VQX0lOU1RPUkUiLCJQSUNLVVBfQ1VSQlNJREUiXX1dLCJzaGlwcGluZ0FkZHJlc3MiOnsibGF0aXR1ZGUiOjQyLjAyODUsImxvbmdpdHVkZSI6LTkzLjY3MzIsInBvc3RhbENvZGUiOiI1MDAxNCIsImNpdHkiOiJBbWVzIiwic3RhdGUiOiJJQSIsImNvdW50cnlDb2RlIjoiVVNBIiwiZ2lmdEFkZHJlc3MiOmZhbHNlfSwiYXNzb3J0bWVudCI6eyJub2RlSWQiOiI3NDkiLCJkaXNwbGF5TmFtZSI6IkFtZXMgU3VwZXJjZW50ZXIiLCJhY2Nlc3NQb2ludHMiOm51bGwsInN1cHBvcnRlZEFjY2Vzc1R5cGVzIjpbXSwiaW50ZW50IjoiUElDS1VQIiwic2NoZWR1bGVFbmFibGVkIjpmYWxzZX0sImRlbGl2ZXJ5Ijp7ImJ1SWQiOiIwIiwibm9kZUlkIjoiNzQ5IiwiZGlzcGxheU5hbWUiOiJBbWVzIFN1cGVyY2VudGVyIiwibm9kZVR5cGUiOiJTVE9SRSIsImFkZHJlc3MiOnsicG9zdGFsQ29kZSI6IjUwMDEwIiwiYWRkcmVzc0xpbmUxIjoiMzEwNSBHcmFuZCBBdmUiLCJjaXR5IjoiQW1lcyIsInN0YXRlIjoiSUEiLCJjb3VudHJ5IjoiVVMiLCJwb3N0YWxDb2RlOSI6IjUwMDEwLTQyMDEifSwiZ2VvUG9pbnQiOnsibGF0aXR1ZGUiOjQyLjA1MzMzNywibG9uZ2l0dWRlIjotOTMuNjIyMDM2fSwiaXNHbGFzc0VuYWJsZWQiOnRydWUsInNjaGVkdWxlZEVuYWJsZWQiOnRydWUsInVuU2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZSwiYWNjZXNzUG9pbnRzIjpbeyJhY2Nlc3NUeXBlIjoiREVMSVZFUllfQUREUkVTUyJ9XSwiaHViTm9kZUlkIjoiNzQ5IiwiaXNFeHByZXNzRGVsaXZlcnlPbmx5IjpmYWxzZSwic3VwcG9ydGVkQWNjZXNzVHlwZXMiOlsiREVMSVZFUllfQUREUkVTUyJdfSwiaW5zdG9yZSI6ZmFsc2UsInJlZnJlc2hBdCI6MTY1ODc5MDg2MTQwMSwidmFsaWRhdGVLZXkiOiJwcm9kOnYyOjg2ZGZmMmRkLThjZTYtNDRhNi05NmViLTc3YzgyNGJkNDdjOSJ9; TS01b0be75=01538efd7ce0b470d64a987ebf3602db685d0723a272fa78f8fac32805933479a0f84d22ed8b2c442be79f7a32c3068c7dd3f30f08; ak_bmsc=EB08BB5A21DDB4B040C846BDE6F20A79~000000000000000000000000000000~YAAQVmDcFysDryOCAQAAjEpaNhAtDFnsSxKun1NnR9PR4dX+nLrhTdYrvw/rP659KYRqKC9qT7YYv9bTVmxBlUdbKzBiBddCCUXIB1dZAciw0LEbnurJiEE/sH8N1UGDz1lOysxIWWZOBoSFSBgIl6oaLnAbtCQlxw8EoaHYlaQmHihodz/iH52O+oODTYtd7yNeMD98wXm0ugPyZRjXj9ycfiO9UF9p/PzyoDsRfw4aw0DvKBAoy0KilaMoJlVJj70oKP9VTx66k0poUoj04Hr5MpHY24qx7DE35Rb1NyZeCN+QyjH461/oLtgCYU0k+1LvHvh/7VBbB/9pMRQi5MtwpPOKFWayEZttz43ejO7nB9ly7lBl5h6Jgsj+chR05+rpFJTk+DqEpT9IKm+x748oCmnqvD4HvS1Gtm9b/0Jv6dNGYbyP4rtuDcccPVDK+DHRXU4q2PqUphk4pPrJhiQ0vQVZdcP4f2A=; _px3=6e284d25985e9047322cf9fc78ece82b368ea3b09daf91b0c2102aac0e1fc32f:OVlTYCsqtIRt4kJ+d7UJDPY8FBtQeUuLFV6sfNKjML10EcrDK4nvQAkexNSIkXOHiIpSSxX1C73pfZCiasSBrw==:1000:56ERtWFsPW8p69z6X+vyTHqzFPdgnB42mHWxrWGDGEeNwiTpYpP/bdrnC3VZGKp0l21/AnO06giIs58+hP+yRLhlvF6gEW6AvN0PK7/Pty3J0ERmgjbMpWXTc/5wQ5yn5ivIqs3I6LFA/sGhuHsXh0ysAz/dR9AMKV62/t2pmoJ2k8TnMK1B2ha6LyeqoTWLCHdGT/TIT8I79+tNfFzwMQ==; TS012768cf=017a255127fd5a6682107dfebf020e084bbb818b80ea418e74b563eb7e180e5c80c84f861f00a3c1f5d32b4c3c58ae254b282aba8c; TS01a90220=017a255127fd5a6682107dfebf020e084bbb818b80ea418e74b563eb7e180e5c80c84f861f00a3c1f5d32b4c3c58ae254b282aba8c; TS2a5e0c5c027=089e03c4ceab2000938f4f25a9ab2df3470831b60c6f0f2da16a220468221195b27a73e267dc6dff088b89bdff113000fc075e5d951bb04f9ddd1f9f810911534fe0a28ce9d271a7456a3f5c60938b0d233223be6d88f691c42ef52bd3379a6d; adblocked=true; akavpau_p2=1658770077~id=769d7ba29e7884f3e66b8f695051849c; com.wm.reflector=\"reflectorid:0000000000000000000000@lastupd:1658769478000@firstcreate:1658759250467\"; xptwg=638832282:B37A1285C727C8:1D0B5D3:84B0816E:A8BC83CB:B5EDE27D:; bm_sv=E4CB63EE2ED3E23722324F4C3970E6BD~YAAQTWDcF6OXIyyCAQAA3ZNdNhC8KYWeQXsqneLTau7gl5gjBronkQhFyXj0MIOG31zcb6d6XyBvWfTkIa3WSA4uJHSVUVqK65cuOYtVCk+8O8e3Y09C3zzHWmnL6o25xph9/gXLuCf3clRePFIIBzHDPca1/lcQMfdZoh1WFlXLjru4mzEjWNchxmMnA97NjcNxWymLVrSUUHALJU51KReGOjHZf733ASgZQmrWx3FhQXxwmKqzrEJJfWcazFz4ipw=~1; TB_SFOU-100=; TS013ed49a=01538efd7c6c84a8bd2593b26d789638b90c07baf2c66119ebc733f3cdfba420d33333db4d80a3afcaddc168ca8a918e233f36b934; TS01a90220=01c561f7cd3e11f7f2082a46d94d5e11394bb9a0c7c8b9da3bbe1bf03ac4467cc34300e6b1304f090ba6b71f3d2d84e18b3702d0a3; bm_sv=E4CB63EE2ED3E23722324F4C3970E6BD~YAAQ22ncF49emyWCAQAAfvhaNhAgBqLM4pTbWVc0n8xtUTEjLnwuiHMht2jhV1teLg+vPbWUzjyo5ch5qii/MlTh2Ztzen4T9i1apBT8JdscBRSwWRvAqiH8D2qN+PyJMK90A8endtKwUeMgH1P/Hu0O/TZ9WUa9rR+H/ceNovnGQ50EX6NouYIDdd+TGq9tDL1LaB726l8WnClHFVclEX/dfPmUkqh3RoUyPhivRHcmg1mDvBYpywMRkboHRFOZ8+Y=~1; bstc=Sj1sMCsRTe4FYrsb-ZJ8zI; com.wm.reflector=\"reflectorid:0000000000000000000000@lastupd:1658769464000@firstcreate:1658759250467\"; exp-ck=0wI6o12SWkj155ZtN1927zv1HDfyl2Iui6D1NPqGB1O1c3v2PzQ_l1Qd0ry1SmVSa1W9IW02Zemho1ccDGr1eEurn1iqBQY1ovdrI3qsDvB1qyn672; mobileweb=0; vtc=TdSHZIomVyLFg458gx4NQk; xpa=0wI6o|1A0pE|2Oqb6|2SWkj|55ZtN|927zv|Ao5F6|CN28l|HDfyl|IOIpg|Iui6D|K2DXb|LTD5Y|NPqGB|O1c3v|PzQ_l|Qd0ry|SmVSa|T-onc|W9IW0|Zemho|aPJIO|bcl64|ccDGr|duBe9|eEurn|elin2|iqBQY|lqVt_|oDpYF|ovdrI|qsDvB|qyn67|sbXp_|uHfV2|uru_L|zCylr; xpm=1%2B1658769258%2BTdSHZIomVyLFg458gx4NQk~%2B0; xptwg=2287528680:7D8C5B1CB6AF18:1451363:C14472AA:CEB8E48C:19BB702E:; TS012768cf=01c561f7cd3e11f7f2082a46d94d5e11394bb9a0c7c8b9da3bbe1bf03ac4467cc34300e6b1304f090ba6b71f3d2d84e18b3702d0a3; TS01b0be75=01538efd7ce0b470d64a987ebf3602db685d0723a272fa78f8fac32805933479a0f84d22ed8b2c442be79f7a32c3068c7dd3f30f08; TS2a5e0c5c027=08f64811c6ab2000f0906850a778d154af1b095199192ed5fb8122aae3616b8d2c8bfeee964123a80871b0075611300001fc5a20e352c887d96725b4e578a166691057807a079614eb81c3996a6509e6fcec7e3b9c6678178a7a8b0f1423e7c9; akavpau_p2=1658769928~id=79dda34b0303cbfc1f64c342d456e046',\n",
        "    'device_profile_ref_id': 'qTKd0T1zkzAfQoH2UwWboYAcvArp5JFf_BhM',\n",
        "    'dnt': '1',\n",
        "    'origin': 'https://www.walmart.com',\n",
        "    'referer': 'https://www.walmart.com/browse/food/fresh-fruits/976759_976793_9756351?affinityOverride=default',\n",
        "    'sec-fetch-dest': 'empty',\n",
        "    'sec-fetch-mode': 'cors',\n",
        "    'sec-fetch-site': 'same-origin',\n",
        "    'sec-gpc': '1',\n",
        "    'traceparent': '4G3H8p-jaQHEmGGZ_ADiYsy83oA0Yr8JGG8o',\n",
        "    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36',\n",
        "    'wm_mp': 'true',\n",
        "    'wm_page_url': 'https://www.walmart.com/browse/food/fresh-fruits/976759_976793_9756351?affinityOverride=default',\n",
        "    'wm_qos.correlation_id': '4G3H8p-jaQHEmGGZ_ADiYsy83oA0Yr8JGG8o',\n",
        "    'x-apollo-operation-name': 'Browse',\n",
        "    'x-enable-server-timing': '1',\n",
        "    'x-latency-trace': '1',\n",
        "    'x-o-bu': 'WALMART-US',\n",
        "    'x-o-ccm': 'server',\n",
        "    'x-o-correlation-id': '4G3H8p-jaQHEmGGZ_ADiYsy83oA0Yr8JGG8o',\n",
        "    'x-o-gql-query': 'query Browse',\n",
        "    'x-o-mart': 'B2C',\n",
        "    'x-o-platform': 'rweb',\n",
        "    'x-o-platform-version': 'main-1.9.0-528d01',\n",
        "    'x-o-segment': 'oaoh'\n",
        "  }\n",
        "\n",
        "  response = requests.request(\"POST\", url, headers=headers, data=payload)\n",
        "\n",
        "#Retreive the relevant information \n",
        "  dt = response.json()\n",
        "  for p in dt['data']['search']['searchResult']['itemStacks'][0]['itemsV2']:\n",
        "    result.append(p)\n",
        "\n",
        "\n",
        "#Save json output into dataframe \n",
        "df = pd.json_normalize(result)"
      ],
      "metadata": {
        "id": "ZNx8kHpeQef3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Drop non-relevant columns\n",
        "\n",
        "df.drop(['__typename', 'fitmentLabel',\n",
        "       'checkStoreAvailabilityATC', 'seeShippingEligibility', 'brand', 'type',\n",
        "       'shortDescription', 'externalInfo', 'itemType',\n",
        "       'classType', 'averageRating', 'numberOfReviews', 'esrb', 'mediaRating',\n",
        "       'salesUnitType', 'sellerId', 'sellerName', 'hasSellerBadge',\n",
        "       'productLocation', 'fulfillmentSpeed', 'offerId', 'variantCriteria',\n",
        "       'snapEligible', 'fulfillmentBadge', 'fulfillmentTitle',\n",
        "       'fulfillmentType', 'manufacturerName', 'showAtc', 'showOptions',\n",
        "       'showBuyNow', 'rewards', 'imageInfo.thumbnailUrl', 'imageInfo.size',\n",
        "       'category.path', 'badges.flags', 'badges.tags',\n",
        "       'availabilityStatusV2.display', 'availabilityStatusV2.value',\n",
        "       'groupMetaData.groupType', 'groupMetaData.groupSubType',\n",
        "       'groupMetaData.numberOfComponents', 'groupMetaData.groupComponents',\n",
        "       'preOrder.isPreOrder', 'preOrder.preOrderMessage',\n",
        "       'preOrder.preOrderStreetDateMessage', 'priceInfo.priceRange',\n",
        "       'priceInfo.shipPrice', 'priceInfo.subscriptionPrice',\n",
        "       'priceInfo.priceDisplayCodes.priceDisplayCondition',\n",
        "       'priceInfo.priceDisplayCodes.submapType', 'sponsoredProduct.spQs',\n",
        "       'sponsoredProduct.clickBeacon', 'sponsoredProduct.spTags',\n",
        "       'sponsoredProduct.viewBeacon', 'sponsoredProduct','moduleLocation', 'lazy',\n",
        "       'priceInfo.priceRange.minPrice', 'priceInfo.priceRange.maxPrice',\n",
        "       'priceInfo.currentPrice'], axis=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6o9LB1E6sXKV",
        "outputId": "63c0af04-3663-4bfe-dc90-58c0b7f70a4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['__typename', 'id', 'usItemId', 'fitmentLabel', 'name',\n",
              "       'checkStoreAvailabilityATC', 'seeShippingEligibility', 'brand', 'type',\n",
              "       'shortDescription', 'canonicalUrl', 'externalInfo', 'itemType',\n",
              "       'classType', 'averageRating', 'numberOfReviews', 'esrb', 'mediaRating',\n",
              "       'salesUnitType', 'sellerId', 'sellerName', 'hasSellerBadge',\n",
              "       'productLocation', 'fulfillmentSpeed', 'offerId', 'variantCriteria',\n",
              "       'snapEligible', 'fulfillmentBadge', 'fulfillmentTitle',\n",
              "       'fulfillmentType', 'manufacturerName', 'showAtc', 'showOptions',\n",
              "       'showBuyNow', 'rewards', 'imageInfo.thumbnailUrl', 'imageInfo.size',\n",
              "       'category.path', 'badges.flags', 'badges.tags',\n",
              "       'availabilityStatusV2.display', 'availabilityStatusV2.value',\n",
              "       'groupMetaData.groupType', 'groupMetaData.groupSubType',\n",
              "       'groupMetaData.numberOfComponents', 'groupMetaData.groupComponents',\n",
              "       'preOrder.isPreOrder', 'preOrder.preOrderMessage',\n",
              "       'preOrder.preOrderStreetDateMessage', 'priceInfo.priceRange',\n",
              "       'priceInfo.currentPrice.price', 'priceInfo.currentPrice.priceString',\n",
              "       'priceInfo.wasPrice', 'priceInfo.unitPrice.price',\n",
              "       'priceInfo.unitPrice.priceString', 'priceInfo.listPrice',\n",
              "       'priceInfo.shipPrice', 'priceInfo.subscriptionPrice',\n",
              "       'priceInfo.priceDisplayCodes.priceDisplayCondition',\n",
              "       'priceInfo.priceDisplayCodes.finalCostByWeight',\n",
              "       'priceInfo.priceDisplayCodes.submapType', 'sponsoredProduct.spQs',\n",
              "       'sponsoredProduct.clickBeacon', 'sponsoredProduct.spTags',\n",
              "       'sponsoredProduct.viewBeacon', 'sponsoredProduct',\n",
              "       'priceInfo.listPrice.price', 'priceInfo.listPrice.priceString',\n",
              "       'priceInfo.unitPrice', 'moduleLocation', 'lazy',\n",
              "       'priceInfo.priceRange.minPrice', 'priceInfo.priceRange.maxPrice',\n",
              "       'priceInfo.currentPrice'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Save to csv\n",
        "df.to_csv('walmart.csv', index=False)"
      ],
      "metadata": {
        "id": "nfLgftIwstGR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Next Steps\n",
        "\n",
        "*Create a function to run the scripts once a day/week\n"
      ],
      "metadata": {
        "id": "1cv600TEukwA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Example\n",
        "\n",
        "# today = date.today(\"%m/%d/%Y\")\n",
        "\n",
        "# schedule.evry(1).friday.do(web_scraping)\n",
        "\n",
        "# while True:\n",
        "#   schedule.run_pending()\n",
        "#   time.sleep(1)"
      ],
      "metadata": {
        "id": "9-OXzRkjuYw6"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}