02. lecturenotes_note_material_api.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Discovering Hidden APIs (Part-2)\n",
    "\n",
    "> Discovering and exploring **lecture note material API** by [lecturenotes.in](https://lecturenotes.in). The API is protected with **CSRF Token** header.\n",
    "\n",
    "![](https://cdn.darknet.org.uk/wp-content/uploads/2017/07/All-You-Need-To-Know-About-Cross-Site-Request-Forgery-CSRF.jpg)\n",
    "\n",
    "## CSRF Token\n",
    "\n",
    "![](https://i.stack.imgur.com/57BlC.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://lecturenotes.in/notes/10913-notes-for-programming-in-c-c-by-bibhuprasad-sahu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = requests.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = sess.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(r.content, 'html5lib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "csrf_token = soup.find('meta', attrs={'name': 'csrf-token'})['content']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_url = \"https://lecturenotes.in/material/10913/page-{}?noOfItems=30\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'x-csrf-token': csrf_token\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = sess.get(api_url.format(31), headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'page': [{'id': '59c178e85ea13239f402b01c',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/31-3897d4622ce6dfd7ef11b08f2a345ca5.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 967,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 31},\n",
       "  {'id': '59c178e85ea13239f402b01d',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/32-65eebdd517f853ea27e24e421ec2cef8.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 974,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 32},\n",
       "  {'id': '59c178e85ea13239f402b01e',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/33-f5b16e39bdc8509f6768c320fd369471.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 969,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 33},\n",
       "  {'id': '59c178e85ea13239f402b01f',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/34-19a125cbca0279d9f7dc23ad1ae5fa18.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 966,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 34},\n",
       "  {'id': '59c178e85ea13239f402b020',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/35-1652b94b9234ed67630ef1ef41ea06e4.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 971,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 35},\n",
       "  {'id': '59c178e85ea13239f402b021',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/36-3ac04efc1b7689a6acdf4fcb4b4cc526.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 973,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 36},\n",
       "  {'id': '59c178e85ea13239f402b022',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/37-f1a811def5ecc9cff8579ed54fc56547.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 971,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 37},\n",
       "  {'id': '59c178e85ea13239f402b023',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/38-71e3a2318909eda22d242282620bade1.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 973,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 38},\n",
       "  {'id': '59c178e85ea13239f402b024',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/39-58383b9679ec9e8857d29ce62448f846.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 972,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 39},\n",
       "  {'id': '59c178e85ea13239f402b025',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/40-ef31c33b499f11c06fa6b826686a14ef.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 974,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 40},\n",
       "  {'id': '59c178e85ea13239f402b028',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/41-cad38ae21bae82248f879017a7d34a78.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 955,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 41},\n",
       "  {'id': '59c178e85ea13239f402b029',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/42-a618cf702088200870b1c89f8d4f2423.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 966,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 42},\n",
       "  {'id': '59c178e85ea13239f402b02a',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/43-d5630ebf06adef5874d4534a46b873c4.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 953,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 43},\n",
       "  {'id': '59c178e85ea13239f402b02b',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/44-e7920c2d37b9770cfab7e10b0be44c8b.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 973,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 44},\n",
       "  {'id': '59c178e85ea13239f402b02c',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/45-e06c65a8205a425c065eca1cb81d47c1.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 962,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 45},\n",
       "  {'id': '59c178e85ea13239f402b02d',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/46-6552c36836743a123a954361c7d1d021.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 971,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 46},\n",
       "  {'id': '59c178e85ea13239f402b02e',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/47-10fcfa39da1534ce97e467ad1094fb2e.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 969,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 47},\n",
       "  {'id': '59c178e85ea13239f402b031',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/48-4715cc0ba3ef667eb5b68fc3a46d705f.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 969,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 48},\n",
       "  {'id': '59c178e85ea13239f402b032',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/49-d2b074fd1d11445b7ec86d13981bfdd0.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 969,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 49},\n",
       "  {'id': '59c178e85ea13239f402b033',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/50-aae4c1b37d1fd8b52b013c27cceff81a.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 970,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 50},\n",
       "  {'id': '59c178e85ea13239f402b034',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/51-56b3f0a3dfa5e2e257bfbbafdfd5da36.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 973,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 51},\n",
       "  {'id': '59c178e85ea13239f402b035',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/52-be35ac814ebb0090022b4558d2b17ef8.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 974,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 52},\n",
       "  {'id': '59c178e85ea13239f402b038',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/53-99ddb39c5f1e5656a3c2478f7f9e2e13.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 972,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 53},\n",
       "  {'id': '59c178e85ea13239f402b039',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/54-aee2b5516eeec8ed5c3fcc9bac752843.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 963,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 54},\n",
       "  {'id': '59c178e85ea13239f402b03a',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/55-9ec90230d5f3e8fc396937d52de31d78.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 962,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 55},\n",
       "  {'id': '59c178e85ea13239f402b03b',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/56-ef314c370651b6a9d0d21a6679cd1937.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 964,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 56},\n",
       "  {'id': '59c178e85ea13239f402b03c',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/57-ab46515a028f5a5f30193fcc8c2e1456.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 973,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 57},\n",
       "  {'id': '59c178e85ea13239f402b03d',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/58-550a3ca3daa765dce01a5a1841b9dbcb.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 969,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 58},\n",
       "  {'id': '59c178e85ea13239f402b03e',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/59-d19abe7deb26123acbe8e00ecc20b90e.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 975,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 59},\n",
       "  {'id': '59c178e85ea13239f402b03f',\n",
       "   'path': '/uploads/upload/note/D2/D2E7XNLj1r/60-040b85d8a2c41883d85d5c7adb14ea0c.jpeg',\n",
       "   'width': 700,\n",
       "   'height': 967,\n",
       "   'text': None,\n",
       "   'type': 'jpg',\n",
       "   'pageNum': 60}]}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in range(1, 101, 30):\n",
    "    r = sess.get(api_url.format(x), headers=headers)\n",
    "    data.extend(r.json()['page'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '59c178e85ea13239f402affa',\n",
       " 'path': '/uploads/upload/note/D2/D2E7XNLj1r/1-6b272ca7a169d207688db6faae4cf48e.jpeg',\n",
       " 'width': 700,\n",
       " 'height': 969,\n",
       " 'text': None,\n",
       " 'type': 'jpg',\n",
       " 'pageNum': 1}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.mkdir(\"lecture\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in data:\n",
    "    image_url = \"https://lecturenotes.in\" + row['path']\n",
    "    r = requests.get(image_url)\n",
    "    \n",
    "    with open(\"lecture/{}.jpg\".format(row['pageNum']), 'wb') as f:\n",
    "        f.write(r.content)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}