Codes/Hyvee_Scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "scrapeHyvee.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOlTwLK5feVf4oMuMrhyTMS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/DSPG-2022/Local-Food-Benchmarking/blob/main/scrapeHyvee.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HpOOuP3vVFDh"
      },
      "outputs": [],
      "source": [
        "#Import libraries\n",
        "\n",
        "!apt-get update \n",
        "!apt install chromium-chromedriver\n",
        "!cp /usr/lib/chromium-browser/chromedriver /usr/bin\n",
        "!pip install selenium\n",
        "!pip install kora -q\n",
        "\n",
        "import time\n",
        "import pandas as pd\n",
        "import requests\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import json"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Follow intructions in [userDoc_autoscraper.pdf](https://github.com/DSPG-2022/Local-Food-Benchmarking/blob/main/notebooks/userDoc_autoscraper.pdf) to use [Postman API](https://www.postman.com/) to retreive the API response from the website."
      ],
      "metadata": {
        "id": "nfFz35SnxJ2Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#API response is replicated by using the Postman API\n",
        "\n",
        "#Fresh Vegetables (Hyvee)\n",
        "\n",
        "# url = \"https://www.hy-vee.com/aisles-online/api/graphql/two-legged/LoadSearchProductsForProductCardsQuery\"\n",
        "\n",
        "# payload = json.dumps({\n",
        "#   \"operationName\": \"LoadSearchProductsForProductCardsQuery\",\n",
        "#   \"variables\": {\n",
        "#     \"targeted\": False,\n",
        "#     \"itemEnabled\": False,\n",
        "#     \"retailItemEnabled\": False,\n",
        "#     \"locationIds\": [],\n",
        "#     \"pageSize\": 60,\n",
        "#     \"pickupLocationHasLocker\": False,\n",
        "#     \"storeId\": 1759,\n",
        "#     \"whereIds\": [\n",
        "#       3354536,\n",
        "#       8679,\n",
        "#       37141,\n",
        "#       8512,\n",
        "#       37565,\n",
        "#       3236281,\n",
        "#       3053604,\n",
        "#       2558,\n",
        "#       37166,\n",
        "#       8463,\n",
        "#       2325575,\n",
        "#       8768,\n",
        "#       10936,\n",
        "#       1698467,\n",
        "#       1698469,\n",
        "#       413679,\n",
        "#       37177,\n",
        "#       93611,\n",
        "#       8689,\n",
        "#       8643,\n",
        "#       2563,\n",
        "#       37313,\n",
        "#       8473,\n",
        "#       100448,\n",
        "#       8694,\n",
        "#       37305,\n",
        "#       8486,\n",
        "#       37288,\n",
        "#       2381276,\n",
        "#       285555,\n",
        "#       3452422,\n",
        "#       37589,\n",
        "#       37569\n",
        "#     ]\n",
        "#   },\n",
        "#   \"query\": \"query LoadSearchProductsForProductCardsQuery($whereIds: [Int!], $storeId: Int!, $pageSize: Int!, $pickupLocationHasLocker: Boolean!, $targeted: Boolean = false, $itemEnabled: Boolean = false, $retailItemEnabled: Boolean = false, $locationIds: [ID!] = []) {\\n  products(pageSize: $pageSize, whereIds: $whereIds) {\\n    products {\\n      ...IProductFragment\\n      __typename\\n    }\\n    __typename\\n  }\\n}\\n\\nfragment IStoreProductFragment on storeProduct {\\n  storeProductId\\n  productId\\n  onFuelSaver\\n  onSale\\n  fuelSaver\\n  price\\n  basePrice\\n  priceMultiple\\n  isWeighted\\n  isActive\\n  isAlcohol\\n  insertDate\\n  departmentId\\n  taxRate\\n  storeProductDescriptions {\\n    type\\n    description\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment ProductImageFragment on productImage {\\n  productId\\n  uri\\n  size\\n  type\\n  isPrimary\\n  __typename\\n}\\n\\nfragment IRetailItemFragment on RetailItem {\\n  retailItemId\\n  basePrice\\n  basePriceQuantity\\n  retailSection {\\n    retailAisle {\\n      sanitizedDescription\\n      __typename\\n    }\\n    __typename\\n  }\\n  soldByUnitOfMeasure {\\n    code\\n    name\\n    __typename\\n  }\\n  tagPrice\\n  tagPriceQuantity\\n  __typename\\n}\\n\\nfragment IItemFragment on Item {\\n  itemId\\n  description\\n  ecommerceStatus\\n  source\\n  images {\\n    imageId\\n    url\\n    isPrimaryImage\\n    __typename\\n  }\\n  source\\n  unitAverageWeight\\n  retailItems(locationIds: $locationIds) @include(if: $retailItemEnabled) {\\n    ...IRetailItemFragment\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment IProductFragment on product {\\n  productId\\n  name\\n  size\\n  averageWeight\\n  upc\\n  brandName\\n  isNotEligibleForDelivery\\n  productImages(where: {isPrimary: true}) {\\n    ...ProductImageFragment\\n    __typename\\n  }\\n  couponProductV4(targeted: $targeted) {\\n    upc\\n    couponsV4 {\\n      couponId\\n      brand\\n      offerState\\n      __typename\\n    }\\n    __typename\\n  }\\n  productLockers @include(if: $pickupLocationHasLocker) {\\n    productLockerId\\n    pickupLocationId\\n    isLockerEligible\\n    __typename\\n  }\\n  storeProduct(storeId: $storeId, isActive: true) {\\n    ...IStoreProductFragment\\n    __typename\\n  }\\n  item @include(if: $itemEnabled) {\\n    ...IItemFragment\\n    __typename\\n  }\\n  __typename\\n}\\n\"\n",
        "# })\n",
        "# headers = {\n",
        "#   'authority': 'www.hy-vee.com',\n",
        "#   'accept': '*/*',\n",
        "#   'accept-language': 'en-US,en;q=0.9',\n",
        "#   'content-type': 'application/json',\n",
        "#   'cookie': '.ASPXANONYMOUS=imtKXPDA2AEkAAAAYmNkOWVmNTItN2Q5OC00MzdmLThmY2MtN2Q1YjFmMGRiNjU0M6ygzLimmd7UY-kBaTVSdhy24_Q1; ASP.NET_SessionId=3pbemvmeyua11wifbztj12jj; unleashSessionId=d184a60c-e5a8-4b84-ac00-ee0499a4b67b; ecomCartID=547815778; ecomCartUUID=a3c796b2-f85c-4791-9b53-e183e1a89ce5; BIGipServer~Web_Administrators~Hy-Vee.com_PROD.app~Hy-Vee.com_PROD_pool=!ORIKI4DCjU/4YZmh8N3h9kfmCpsNVQwtdd3gr5qvN9NU/XNvA+JQZi7DibnTziNQEZUn6CKwUkzggg==; _gcl_au=1.1.1260926635.1656359159; _ga=GA1.1.2067504075.1656359159; mdLogger=false; kampyle_userid=b454-33cd-66ec-dc3d-0709-0054-4263-51bd; kampyleUserSession=1656359159309; kampyleUserSessionsCount=1; kampyleUserPercentile=84.23550293565494; navigation=aisles; _wc_cust_guid_=1a686292-f3af-43bd-ef37-a1c11d78be0a; __cf_bm=08wXTiC2FxUerlHJmUDObHe1Lp3fz8iQiFwJQgHKu.A-1656367806-0-Afs4TCwizBox4FYlSaQ7Db2NoHIOyE14ShpZKHWQ4/DbBATFjOUvEboY4k0iDfj4ny5QxQQ4cZiMCIGLHl9MvUc=; kampyleSessionPageCounter=20; aisles-online-state=Fe26.2**7eb4342bd06135c88b87f721c33c1da255535ebeaa2cdda7a895b3f12c2d22c5*6SpmnmVD65nwo1btpa6eUA*nXpRJI-hwY8a5z7-4y4pgKCy1NalVqq1zkZNCcru2MG-KZuM9OblfOoIyXyx_2jK**f225f152958af69335618d47ffb0faa477d9b3a80769f1eea471275880e02b0e*G_aGW33g5NepjNoZZKsYYyR2U0lavDRzPs0RnsfQPA4; _ga_JJP2YPFRE4=GS1.1.1656365441.2.1.1656368021.0',\n",
        "#   'dnt': '1',\n",
        "#   'origin': 'https://www.hy-vee.com',\n",
        "#   'referer': 'https://www.hy-vee.com/aisles-online/browse/fresh/fresh-fruits-and-vegetables/fresh-fruits?page=3',\n",
        "#   'sec-ch-ua': '\" Not;A Brand\";v=\"99\", \"Microsoft Edge\";v=\"103\", \"Chromium\";v=\"103\"',\n",
        "#   'sec-ch-ua-mobile': '?0',\n",
        "#   'sec-ch-ua-platform': '\"Windows\"',\n",
        "#   'sec-fetch-dest': 'empty',\n",
        "#   'sec-fetch-mode': 'cors',\n",
        "#   'sec-fetch-site': 'same-origin',\n",
        "#   'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'\n",
        "# }\n",
        "\n",
        "#Fresh vege - Hyvee\n",
        "\n",
        "import requests\n",
        "import json\n",
        "\n",
        "url = \"https://www.hy-vee.com/aisles-online/api/graphql/two-legged/LoadSearchProductsForProductCardsQuery\"\n",
        "\n",
        "payload = json.dumps({\n",
        "  \"operationName\": \"LoadSearchProductsForProductCardsQuery\",\n",
        "  \"variables\": {\n",
        "    \"targeted\": False,\n",
        "    \"itemEnabled\": False,\n",
        "    \"retailItemEnabled\": False,\n",
        "    \"locationIds\": [],\n",
        "    \"pageSize\": 60,\n",
        "    \"pickupLocationHasLocker\": False,\n",
        "    \"storeId\": 1759,\n",
        "    \"whereIds\": [\n",
        "      2553,\n",
        "      37358,\n",
        "      3413624,\n",
        "      8805,\n",
        "      8819,\n",
        "      10979,\n",
        "      11278,\n",
        "      19472,\n",
        "      107891,\n",
        "      3057184,\n",
        "      8503,\n",
        "      1351403,\n",
        "      8799,\n",
        "      338788,\n",
        "      3027111,\n",
        "      37441,\n",
        "      93506,\n",
        "      65597,\n",
        "      1891422,\n",
        "      37574,\n",
        "      8841,\n",
        "      2665164,\n",
        "      1891421,\n",
        "      1770951,\n",
        "      3327180,\n",
        "      1891420,\n",
        "      10959,\n",
        "      3327176,\n",
        "      93627,\n",
        "      93534,\n",
        "      8840,\n",
        "      91694,\n",
        "      8827,\n",
        "      1730136,\n",
        "      639777,\n",
        "      47021,\n",
        "      1693532\n",
        "    ]\n",
        "  },\n",
        "  \"query\": \"query LoadSearchProductsForProductCardsQuery($whereIds: [Int!], $storeId: Int!, $pageSize: Int!, $pickupLocationHasLocker: Boolean!, $targeted: Boolean = false, $itemEnabled: Boolean = false, $retailItemEnabled: Boolean = false, $locationIds: [ID!] = []) {\\n  products(pageSize: $pageSize, whereIds: $whereIds) {\\n    products {\\n      ...IProductFragment\\n      __typename\\n    }\\n    __typename\\n  }\\n}\\n\\nfragment IStoreProductFragment on storeProduct {\\n  storeProductId\\n  productId\\n  onFuelSaver\\n  onSale\\n  fuelSaver\\n  price\\n  basePrice\\n  priceMultiple\\n  isWeighted\\n  isActive\\n  isAlcohol\\n  insertDate\\n  departmentId\\n  taxRate\\n  storeProductDescriptions {\\n    type\\n    description\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment ProductImageFragment on productImage {\\n  productId\\n  uri\\n  size\\n  type\\n  isPrimary\\n  __typename\\n}\\n\\nfragment IRetailItemFragment on RetailItem {\\n  retailItemId\\n  basePrice\\n  basePriceQuantity\\n  retailSection {\\n    retailAisle {\\n      sanitizedDescription\\n      __typename\\n    }\\n    __typename\\n  }\\n  soldByUnitOfMeasure {\\n    code\\n    name\\n    __typename\\n  }\\n  tagPrice\\n  tagPriceQuantity\\n  __typename\\n}\\n\\nfragment IItemFragment on Item {\\n  itemId\\n  description\\n  ecommerceStatus\\n  source\\n  images {\\n    imageId\\n    url\\n    isPrimaryImage\\n    __typename\\n  }\\n  source\\n  unitAverageWeight\\n  retailItems(locationIds: $locationIds) @include(if: $retailItemEnabled) {\\n    ...IRetailItemFragment\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment IProductFragment on product {\\n  productId\\n  name\\n  size\\n  averageWeight\\n  upc\\n  brandName\\n  isNotEligibleForDelivery\\n  productImages(where: {isPrimary: true}) {\\n    ...ProductImageFragment\\n    __typename\\n  }\\n  couponProductV4(targeted: $targeted) {\\n    upc\\n    couponsV4 {\\n      couponId\\n      brand\\n      offerState\\n      __typename\\n    }\\n    __typename\\n  }\\n  productLockers @include(if: $pickupLocationHasLocker) {\\n    productLockerId\\n    pickupLocationId\\n    isLockerEligible\\n    __typename\\n  }\\n  storeProduct(storeId: $storeId, isActive: true) {\\n    ...IStoreProductFragment\\n    __typename\\n  }\\n  item @include(if: $itemEnabled) {\\n    ...IItemFragment\\n    __typename\\n  }\\n  __typename\\n}\\n\"\n",
        "})\n",
        "headers = {\n",
        "  'authority': 'www.hy-vee.com',\n",
        "  'accept': '*/*',\n",
        "  'accept-language': 'en-US,en;q=0.9',\n",
        "  'content-type': 'application/json',\n",
        "  'cookie': '.ASPXANONYMOUS=C02wVWnC2AEkAAAAYjVhZGRhMjUtM2ZiNS00ZTI2LWI4NDgtMWQ0NGI0NjQ1ZTg45eWuNIzEa37u6kZFZcQcdBS1o3E1; ASP.NET_SessionId=23ugpolauv0k2tgcj0bystbq; unleashSessionId=d57f295e-2550-4024-b950-2244b942a791; ecomCartID=548599410; ecomCartUUID=d171572a-f079-429b-8874-5f5df418e785; BIGipServer~Web_Administrators~Hy-Vee.com_PROD.app~Hy-Vee.com_PROD_pool=!a01ulqK79iTbEsGh8N3h9kfmCpsNVfHJrx+Ycfr1IIoyigai+JPCW9y8GJ8sSb4dBIbQB0kUrCGSqA==; navigation=aisles; aisles-online-state=Fe26.2**8d9507ec109af702dfe3b7b2f8fd7738dcd1faf69535325cbf184ec0a9de6d5d*systKEciqPu-A8-klEMIqw*MMi-3gjMd3G9f39Y-TOBWvcP0SPXHcd-35US8LLzPMYASJy-HlqmR-qQF6L_HQhB**0b5c6b215cb369b48421d78b1e15bb0a2563dbd5f31addbe4bbb3661af2964a5*zSABpAkfinVGbnhnZJ1bsNLJ7I3DyTbCq1XQThrxo6c; __cf_bm=dMy3Ije7X3RNbB8ImhgvPpd66uZhLZQgymgBe86ddTU-1656523628-0-Ae8foyplWBLTxQrFhEXr6YVHX6doVGcsmb7wvSNCRoXtaFAAgmrsJfli/ItaRI/Yy5V1Gjce/J72LZTqKUTAZYVxXj2K5xB/kmMcLMW6iwb4LaillOuE2gVVrjqObc6uBqJVL5yYI9O8nqtcn8jgAWjZ76UwygghbnToI/YsZvxk; __cf_bm=tJxct57mhLjE743e7JPHfvT4_p71esWv9tDc5W7WvEk-1656523398-0-ARPWUncC7Vh+uCDAT9NtdpXbuj9Rm5rCqKdPuJg3fNPpKZVfRS8ESRX+s/Q/A3VQuz0ufEy7A5ib9Xl73e3xaVg=',\n",
        "  'dnt': '1',\n",
        "  'origin': 'https://www.hy-vee.com',\n",
        "  'referer': 'https://www.hy-vee.com/aisles-online/browse/fresh/fresh-fruits-and-vegetables/fresh-vegetables?page=4',\n",
        "  'sec-fetch-dest': 'empty',\n",
        "  'sec-fetch-mode': 'cors',\n",
        "  'sec-fetch-site': 'same-origin',\n",
        "  'sec-gpc': '1',\n",
        "  'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.66 Safari/537.36'\n",
        "}\n",
        "\n",
        "response = requests.request(\"POST\", url, headers=headers, data=payload)\n",
        "\n",
        "#print(response.text)\n",
        "\n",
        "dt = json.loads(response.text)\n",
        "\n",
        "print(dt.items())"
      ],
      "metadata": {
        "id": "hTYxlcTHVW_o"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Transform JSON data into a pandas dataframe for analysis\n",
        "\n",
        "df_vege = pd.DataFrame(columns=['name', 'price', 'url', 'id'])\n",
        "\n",
        "  dt = response.json()\n",
        "  for p in dt['data']['products']:\n",
        "    result.append(p)\n",
        "\n",
        "df = pd.json_normalize(result)"
      ],
      "metadata": {
        "id": "7FzsZ-RtVgu3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#View first few rows\n",
        "df.head()\n",
        "\n",
        "#Save to csv\n",
        "df.to_csv('Hyvee.csv', index=False)"
      ],
      "metadata": {
        "id": "ij64tUtsVj11"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}