notebooks/p1_data_api_v1_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T15:21:58.108585Z",
     "start_time": "2020-09-26T15:21:58.104913Z"
    }
   },
   "source": [
    "- This Jupyter notebook is an example of the how to access the RestAPI interface, described at:\n",
    "  https://doc.particle.one/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Credentials / Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:02.908047Z",
     "start_time": "2020-09-26T16:02:02.597546Z"
    }
   },
   "outputs": [],
   "source": [
    "# Make sure you have `pandas` installed.\n",
    "# If you don't have it installed, install it in your usual way (with pip, conda)\n",
    "\n",
    "#! pip install requests pandas\n",
    "\n",
    "import json\n",
    "\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:02.912649Z",
     "start_time": "2020-09-26T16:02:02.910084Z"
    }
   },
   "outputs": [],
   "source": [
    "# Enter your token here.\n",
    "# You can get your token by signing up at `www.particle.one`.\n",
    "#TOKEN = \"YOUR_TOKEN_HERE\"\n",
    "# An example token is like:\n",
    "TOKEN = \"e44e7c6b04ef3ea1cfb7a8a67db74751c177259e\"\n",
    "\n",
    "API_URL = \"https://data.particle.one\"\n",
    "HEADERS = {\"Authorization\": f\"Token {TOKEN}\", \"Content-Type\": \"application/json\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search query structure\n",
    "\n",
    "Search query is a Python `dict` with the following structure:\n",
    "```python\n",
    "query = {\n",
    "    \"text\": \"\",\n",
    "    \"commodity\": [],\n",
    "    \"business_category\": \"\",\n",
    "    \"country\": [],\n",
    "    \"frequency\": []\n",
    "}\n",
    "```\n",
    "The fields are:\n",
    "- `text`: string. Works as a filter. Free text. Everything that have no match with this phrase will be filtered out.\n",
    "- `commodity`: list of strings. Works as a filter. You can find valid values in paragraph 7.1 of this notebook.\n",
    "- `business_category`: string. Works as a filter. You can find valid values in paragraph 7.2 of this notebook.\n",
    "- `country`: list of strings. Works as a filter. You can find valid values in paragraph 7.3 of this notebook.\n",
    "- `frequency`: list of strings. Works as a filter. You can find valid values in paragraph 7.4 of this notebook.\n",
    "\n",
    "Combination of fields work with logical operator AND.\n",
    "E.g. you will get all records that satisfy all filters.\n",
    "\n",
    "`text` **AND** `commodity` **AND** `business_category` **AND** `country` **AND** `frequency`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# POST data-api/v1/search-count/\n",
    "Returns count for the given query."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:02.917036Z",
     "start_time": "2020-09-26T16:02:02.914389Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count_url= https://data.particle.one/data-api/v1/search-count/\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "count_url = os.path.join(API_URL, \"data-api/v1/search-count/\")\n",
    "print(\"count_url=\", count_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:02.921616Z",
     "start_time": "2020-09-26T16:02:02.918888Z"
    }
   },
   "outputs": [],
   "source": [
    "# Prepare query.\n",
    "query = {\n",
    "    \"text\": \"\",\n",
    "    \"commodity\": [\"Corn\"],\n",
    "    \"business_category\": \"\",\n",
    "    \"country\": [],\n",
    "    \"frequency\": [],\n",
    "}\n",
    "payload = json.dumps(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:03.133553Z",
     "start_time": "2020-09-26T16:02:02.923039Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data= {'message': 'OK', 'count': 6}\n"
     ]
    }
   ],
   "source": [
    "# Perform query.\n",
    "response = requests.request(\"POST\", count_url, headers=HEADERS, data=payload)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data=\", data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# POST data-api/v1/search/\n",
    "\n",
    "- It returns the first chunk of the payload metadata for the given query, where a chunk is 1000 records.\n",
    "- It also returns `scroll_id` to get the next portion of the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:03.139889Z",
     "start_time": "2020-09-26T16:02:03.136962Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "search_url= https://data.particle.one/data-api/v1/search/\n"
     ]
    }
   ],
   "source": [
    "search_url = os.path.join(API_URL, \"data-api/v1/search/\")\n",
    "print(\"search_url=\", search_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:03.145207Z",
     "start_time": "2020-09-26T16:02:03.142436Z"
    }
   },
   "outputs": [],
   "source": [
    "# Prepare query.\n",
    "query = {\n",
    "    \"text\": \"Gas\",\n",
    "    \"commodity\": [],\n",
    "    \"business_category\": \"\",\n",
    "    \"country\": [],\n",
    "    \"frequency\": [],\n",
    "}\n",
    "payload = json.dumps(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:04.645368Z",
     "start_time": "2020-09-26T16:02:03.147029Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'scroll_id', 'total_count', 'rows']\n",
      "total_count= 29545\n",
      "scroll_id= DXF1ZXJ5QW5kRmV0Y2gBAAAAAABDhIkWQUFleVN5Ym9RT0daMWhMQU1KNXJ4dw==\n",
      "df.shape= (1000, 8)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>commodity</th>\n",
       "      <th>payload_id</th>\n",
       "      <th>business_category</th>\n",
       "      <th>country</th>\n",
       "      <th>frequency</th>\n",
       "      <th>unit</th>\n",
       "      <th>start_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Actual  Market  Prices: World: Natural  Gas: I...</td>\n",
       "      <td>Liquefied Natural Gas</td>\n",
       "      <td>1632886ae0ee7c8e785769009c1a276f2abf256a</td>\n",
       "      <td>Upstream</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>USD/MBTU</td>\n",
       "      <td>1992-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Actual  Market  Prices: World: Natural  Gas: I...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>1632886ae0ee7c8e785769009c1a276f2abf256a</td>\n",
       "      <td>Upstream</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>USD/MBTU</td>\n",
       "      <td>1992-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Actual  Market  Prices: World: Natural  Gas: R...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>dded2f042b2c773790ececfb7565aa1db8b0992c</td>\n",
       "      <td>Upstream</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>USD/MBTU</td>\n",
       "      <td>1985-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Actual  Market  Prices: World: Natural  Gas: U...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>446c3a3dc964bb48bf88deaf069e59e52b3b057b</td>\n",
       "      <td>Upstream</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Monthly</td>\n",
       "      <td>USD/MBTU</td>\n",
       "      <td>1991-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Alamo – TX Natural Gas Pipeline Exports to Mex...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>dff30995669368dcd485c632c0dccc8188f47fff</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Mexico</td>\n",
       "      <td>Annual</td>\n",
       "      <td>USD/thousand CF</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                name              commodity  \\\n",
       "0  Actual  Market  Prices: World: Natural  Gas: I...  Liquefied Natural Gas   \n",
       "1  Actual  Market  Prices: World: Natural  Gas: I...            Natural Gas   \n",
       "2  Actual  Market  Prices: World: Natural  Gas: R...            Natural Gas   \n",
       "3  Actual  Market  Prices: World: Natural  Gas: U...            Natural Gas   \n",
       "4  Alamo – TX Natural Gas Pipeline Exports to Mex...            Natural Gas   \n",
       "\n",
       "                                 payload_id business_category    country  \\\n",
       "0  1632886ae0ee7c8e785769009c1a276f2abf256a          Upstream  Undefined   \n",
       "1  1632886ae0ee7c8e785769009c1a276f2abf256a          Upstream  Undefined   \n",
       "2  dded2f042b2c773790ececfb7565aa1db8b0992c          Upstream  Undefined   \n",
       "3  446c3a3dc964bb48bf88deaf069e59e52b3b057b          Upstream  Undefined   \n",
       "4  dff30995669368dcd485c632c0dccc8188f47fff         Undefined     Mexico   \n",
       "\n",
       "  frequency             unit  start_date  \n",
       "0   Monthly         USD/MBTU  1992-01-01  \n",
       "1   Monthly         USD/MBTU  1992-01-01  \n",
       "2   Monthly         USD/MBTU  1985-01-01  \n",
       "3   Monthly         USD/MBTU  1991-01-01  \n",
       "4    Annual  USD/thousand CF        None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "response = requests.request(\"POST\", search_url, headers=HEADERS, data=payload)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "\n",
    "assert \"detail\" not in data, data\n",
    "\n",
    "print(\"total_count=\", data[\"total_count\"])\n",
    "\n",
    "# Saving scroll_id for the next query.\n",
    "scroll_id = data[\"scroll_id\"]\n",
    "print(\"scroll_id=\", scroll_id)\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"rows\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GET data-api/v1/search-scroll/?scroll_id="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:04.649197Z",
     "start_time": "2020-09-26T16:02:04.646656Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "search_scroll_url= https://data.particle.one/data-api/v1/search-scroll/?scroll_id=DXF1ZXJ5QW5kRmV0Y2gBAAAAAABDhIkWQUFleVN5Ym9RT0daMWhMQU1KNXJ4dw==\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "\n",
    "# We use scroll id from the previous query.\n",
    "search_scroll_url = os.path.join(\n",
    "    API_URL, f\"data-api/v1/search-scroll/?scroll_id={scroll_id}\"\n",
    ")\n",
    "print(\"search_scroll_url=\", search_scroll_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.010203Z",
     "start_time": "2020-09-26T16:02:04.650491Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'scroll_id', 'rows']\n",
      "data['rows'][0]= {'name': 'Dry natural gas consumption – Barbados – billion cubic meters', 'commodity': 'Natural Gas', 'payload_id': '6eeb38e07d5a5c4b9c7b7d0ee06ba83eff7fb981', 'business_category': 'Undefined', 'country': 'Barbados', 'frequency': 'Annual', 'unit': 'kwh, in billions', 'start_date': None}\n",
      "df.shape= (1000, 8)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>commodity</th>\n",
       "      <th>payload_id</th>\n",
       "      <th>business_category</th>\n",
       "      <th>country</th>\n",
       "      <th>frequency</th>\n",
       "      <th>unit</th>\n",
       "      <th>start_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Dry natural gas consumption – Barbados – billi...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>6eeb38e07d5a5c4b9c7b7d0ee06ba83eff7fb981</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Barbados</td>\n",
       "      <td>Annual</td>\n",
       "      <td>kwh, in billions</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dry natural gas consumption – Barbados – milli...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>05780468618459e59450219200206e8b17463050</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Barbados</td>\n",
       "      <td>Annual</td>\n",
       "      <td>BTU/person, in millions</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dry natural gas consumption – Barbados – quadr...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>4f15dcf120eec1a39546c3b65e0f392ceb09495c</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Barbados</td>\n",
       "      <td>Annual</td>\n",
       "      <td>BTU, in quadrillions</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dry natural gas consumption – Barbados – teraj...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>fab1d3dc9a3236a3abaeedf09b394788f0de21fa</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Barbados</td>\n",
       "      <td>Annual</td>\n",
       "      <td>terajoules</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dry natural gas consumption – Belarus – billio...</td>\n",
       "      <td>Natural Gas</td>\n",
       "      <td>5960c89357da83583b5dd2a85c59678fdc464e4e</td>\n",
       "      <td>Undefined</td>\n",
       "      <td>Belarus</td>\n",
       "      <td>Annual</td>\n",
       "      <td>USD at PPP, in billions</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                name    commodity  \\\n",
       "0  Dry natural gas consumption – Barbados – billi...  Natural Gas   \n",
       "1  Dry natural gas consumption – Barbados – milli...  Natural Gas   \n",
       "2  Dry natural gas consumption – Barbados – quadr...  Natural Gas   \n",
       "3  Dry natural gas consumption – Barbados – teraj...  Natural Gas   \n",
       "4  Dry natural gas consumption – Belarus – billio...  Natural Gas   \n",
       "\n",
       "                                 payload_id business_category   country  \\\n",
       "0  6eeb38e07d5a5c4b9c7b7d0ee06ba83eff7fb981         Undefined  Barbados   \n",
       "1  05780468618459e59450219200206e8b17463050         Undefined  Barbados   \n",
       "2  4f15dcf120eec1a39546c3b65e0f392ceb09495c         Undefined  Barbados   \n",
       "3  fab1d3dc9a3236a3abaeedf09b394788f0de21fa         Undefined  Barbados   \n",
       "4  5960c89357da83583b5dd2a85c59678fdc464e4e         Undefined   Belarus   \n",
       "\n",
       "  frequency                     unit start_date  \n",
       "0    Annual         kwh, in billions       None  \n",
       "1    Annual  BTU/person, in millions       None  \n",
       "2    Annual     BTU, in quadrillions       None  \n",
       "3    Annual               terajoules       None  \n",
       "4    Annual  USD at PPP, in billions       None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "\n",
    "response = requests.request(\"GET\", search_scroll_url, headers=HEADERS)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "print(\"data['rows'][0]=\", data[\"rows\"][0])\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"rows\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GET data-api/v1/payload/?payload_id=\n",
    "Returns payload for the given `payload_id`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.014537Z",
     "start_time": "2020-09-26T16:02:06.011924Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload_url= https://data.particle.one/data-api/v1/payload/?payload_id=8f26ba4734df3a62352cce9d64987d64da54b400\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "\n",
    "# We use one of the `payload_id` from one of the previous queries.\n",
    "payload_id = \"8f26ba4734df3a62352cce9d64987d64da54b400\"\n",
    "payload_url = os.path.join(\n",
    "    API_URL, f\"data-api/v1/payload/?payload_id={payload_id}\"\n",
    ")\n",
    "print(\"payload_url=\", payload_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.150444Z",
     "start_time": "2020-09-26T16:02:06.015979Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'payload_data']\n",
      "df.shape= (48, 4)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_period</th>\n",
       "      <th>original_value</th>\n",
       "      <th>period</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1980-01-01T00:00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1981</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1981-01-01T00:00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1982-01-01T00:00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1983</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1983-01-01T00:00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1984</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1984-01-01T00:00:00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  original_period original_value               period value\n",
       "0            1980            0.0  1980-01-01T00:00:00     0\n",
       "1            1981            0.0  1981-01-01T00:00:00     0\n",
       "2            1982            0.0  1982-01-01T00:00:00     0\n",
       "3            1983            0.0  1983-01-01T00:00:00     0\n",
       "4            1984            0.0  1984-01-01T00:00:00     0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "response = requests.request(\"GET\", payload_url, headers=HEADERS)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"payload_data\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helpers\n",
    "\n",
    "To search we use several predefined lists of names for each parameter of metadata. Such as:\n",
    "- `commodity`\n",
    "- `business-category`\n",
    "- `country`\n",
    "- `frequency`\n",
    "\n",
    "Each parameter has its own set of valid names."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GET data-api/v1/commodities/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.154839Z",
     "start_time": "2020-09-26T16:02:06.152055Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "commodities_url= https://data.particle.one/data-api/v1/commodities/\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "commodities_url = os.path.join(API_URL, \"data-api/v1/commodities/\")\n",
    "print(\"commodities_url=\", commodities_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.267789Z",
     "start_time": "2020-09-26T16:02:06.156589Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'data']\n",
      "df.shape= (61, 1)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Aluminum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Benzene</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Biodiesel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Biofuel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Butadiene</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        name\n",
       "0   Aluminum\n",
       "1    Benzene\n",
       "2  Biodiesel\n",
       "3    Biofuel\n",
       "4  Butadiene"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "response = requests.request(\"GET\", commodities_url, headers=HEADERS)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"data\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GET data-api/v1/business-categories/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.271975Z",
     "start_time": "2020-09-26T16:02:06.269376Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bc_url= https://data.particle.one/data-api/v1/business-categories/\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "bc_url = os.path.join(API_URL, \"data-api/v1/business-categories/\")\n",
    "print(\"bc_url=\", bc_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.376841Z",
     "start_time": "2020-09-26T16:02:06.273745Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'data']\n",
      "df.shape= (4, 1)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Downstream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Midstream</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Undefined</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Upstream</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         name\n",
       "0  Downstream\n",
       "1   Midstream\n",
       "2   Undefined\n",
       "3    Upstream"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "\n",
    "response = requests.request(\"GET\", bc_url, headers=HEADERS)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"data\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GET data-api/v1/countries/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.381303Z",
     "start_time": "2020-09-26T16:02:06.378264Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "countries_url= https://data.particle.one/data-api/v1/countries/\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "countries_url = os.path.join(API_URL, \"data-api/v1/countries/\")\n",
    "print(\"countries_url=\", countries_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.565126Z",
     "start_time": "2020-09-26T16:02:06.385784Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'data']\n",
      "df.shape= (231, 1)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Afghanistan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Albania</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Algeria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>American Samoa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Andorra</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             name\n",
       "0     Afghanistan\n",
       "1         Albania\n",
       "2         Algeria\n",
       "3  American Samoa\n",
       "4         Andorra"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "response = requests.request(\"GET\", countries_url, headers=HEADERS)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"data\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GET data-api/v1/frequencies/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.568910Z",
     "start_time": "2020-09-26T16:02:06.566714Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://data.particle.one/data-api/v1/frequencies/\n"
     ]
    }
   ],
   "source": [
    "# Build entrypoint url.\n",
    "frequencies_url = os.path.join(API_URL, \"data-api/v1/frequencies/\")\n",
    "print(frequencies_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-26T16:02:06.676954Z",
     "start_time": "2020-09-26T16:02:06.570319Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.keys()= ['message', 'data']\n",
      "df.shape= (6, 1)\n",
      "df.head()=\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Daily</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Weekly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Quarterly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Semi-annual</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name\n",
       "0        Daily\n",
       "1       Weekly\n",
       "2      Monthly\n",
       "3    Quarterly\n",
       "4  Semi-annual"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform query.\n",
    "response = requests.request(\"GET\", frequencies_url, headers=HEADERS)\n",
    "data = json.loads(response.text.encode(\"utf8\"))\n",
    "print(\"data.keys()=\", list(data.keys()))\n",
    "\n",
    "df = pd.DataFrame.from_records(data[\"data\"])\n",
    "print(\"df.shape=\", df.shape)\n",
    "print(\"df.head()=\")\n",
    "display(df.head())"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "text_representation": {
    "extension": ".py",
    "format_name": "percent",
    "format_version": "1.3",
    "jupytext_version": "1.4.2"
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}