BBC_Location_ID_Weather_Scraping.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Animeshsinghiit/Web-Scraping-Practises/blob/main/BBC_Location_ID_Weather_Scraping.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WsVuzd8zBh1h"
      },
      "source": [
        "# A tutorial to scrape the location ID of any city in BBC Weather. \n",
        "This code snippet takes city name as input and it hits the BBC Weahter API with a request for location ID. This location ID is used as input in the next part of the code to scrape weather forecast for the city using this location ID.  \n",
        "\n",
        "\n",
        "*Web scraping might not be legal always. It is a good idea to check the terms of the website you plan to scrape before proceeding. Also, if your code requests a url from a server multiple times, it is a good practice to either cache your requests, or insert a timed delay between consecutive requests.*"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "iYX9uq5g11Xt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FVSmmYGpCBvc"
      },
      "source": [
        "import os\n",
        "\n",
        "import requests               # to get the webpage\n",
        "import json                   # to convert API to json format\n",
        "\n",
        "from urllib.parse import urlencode\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re                     # regular expression operators"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XK0bYqe9DAFs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "59b34bd8-5ea0-4657-e1f3-dbf8e79aeba4"
      },
      "source": [
        "test_city = \"New York\"\n",
        "location_url = 'https://locator-service.api.bbci.co.uk/locations?' + urlencode({\n",
        "   'api_key': 'AGbFAKx58hyjQScCXIYrxuEwJh2W2cmv',\n",
        "   's': test_city,\n",
        "   'stack': 'aws',\n",
        "   'locale': 'en',\n",
        "   'filter': 'international',\n",
        "   'place-types': 'settlement,airport,district',\n",
        "   'order': 'importance',\n",
        "   'a': 'true',\n",
        "   'format': 'json'\n",
        "})\n",
        "location_url"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'https://locator-service.api.bbci.co.uk/locations?api_key=AGbFAKx58hyjQScCXIYrxuEwJh2W2cmv&s=New+York&stack=aws&locale=en&filter=international&place-types=settlement%2Cairport%2Cdistrict&order=importance&a=true&format=json'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fXGZdiUDPTs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "37ee84a1-0f3c-41f6-deef-485ece0f9902"
      },
      "source": [
        "result = requests.get(location_url).json()\n",
        "result"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'response': {'results': {'results': [{'container': 'United States',\n",
              "     'containerId': 6252001,\n",
              "     'country': 'US',\n",
              "     'id': '5128581',\n",
              "     'language': 'en',\n",
              "     'latitude': 40.71427,\n",
              "     'longitude': -74.00597,\n",
              "     'name': 'New York',\n",
              "     'placeType': 'settlement',\n",
              "     'timezone': 'America/New_York'},\n",
              "    {'container': 'Lincolnshire',\n",
              "     'containerId': 2644486,\n",
              "     'country': 'GB',\n",
              "     'id': '2641508',\n",
              "     'language': 'en',\n",
              "     'latitude': 53.07897,\n",
              "     'longitude': -0.14008,\n",
              "     'name': 'New York',\n",
              "     'placeType': 'settlement',\n",
              "     'timezone': 'Europe/London',\n",
              "     'topicId': 'cr0mejz7n26t'}],\n",
              "   'totalResults': 2}}}"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rXM4oYcwDgzC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "0a5bf472-2822-4165-fc17-5394899556b3"
      },
      "source": [
        "# Print location id\n",
        "result['response']['results']['results'][0]['id']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'5128581'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z7-r0mgnDrS2"
      },
      "source": [
        "# Creating a function to output location id by taking any city name as input."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JvWzOsPVBhmX"
      },
      "source": [
        "def getlocid(city):\n",
        "    city = city.lower() # convert city name to lowercase to standardize format\n",
        "    # Convert into an API call using URL encoding\n",
        "    location_url = 'https://locator-service.api.bbci.co.uk/locations?' + urlencode({\n",
        "      'api_key': 'AGbFAKx58hyjQScCXIYrxuEwJh2W2cmv',\n",
        "      's': city,\n",
        "      'stack': 'aws',\n",
        "      'locale': 'en',\n",
        "      'filter': 'international',\n",
        "      'place-types': 'settlement,airport,district',\n",
        "      'order': 'importance',\n",
        "      'a': 'true',\n",
        "      'format': 'json'\n",
        "    })\n",
        "    result = requests.get(location_url).json()\n",
        "    locid = result['response']['results']['results'][0]['id']\n",
        "    return locid"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KEiuQasXBhal",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c97749ff-5acf-45f8-d04e-2b0c957a27ef"
      },
      "source": [
        "getlocid('Toronto')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'6167865'"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "deuhR31JleOH"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    }
  ]
}