Ask_A_Pdf_Questions.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9d615a77",
        "outputId": "649ed3f5-6d0e-4d46-e4b8-aecfb2478c0f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.0.175-py3-none-any.whl (872 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m872.8/872.8 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.10)\n",
            "Collecting aiohttp<4.0.0,>=3.8.3 (from langchain)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m59.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0.0,>=4.0.0 (from langchain)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.5.7-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.4)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.22.4)\n",
            "Collecting openapi-schema-pydantic<2.0,>=1.2 (from langchain)\n",
            "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.7)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.27.1)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m25.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.3.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
            "Collecting typing-inspect>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.8.0-py3-none-any.whl (8.7 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<2,>=1->langchain) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: mypy-extensions, multidict, marshmallow, frozenlist, async-timeout, yarl, typing-inspect, openapi-schema-pydantic, marshmallow-enum, aiosignal, dataclasses-json, aiohttp, langchain\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 dataclasses-json-0.5.7 frozenlist-1.3.3 langchain-0.0.175 marshmallow-3.19.0 marshmallow-enum-1.5.1 multidict-6.0.4 mypy-extensions-1.0.0 openapi-schema-pydantic-1.2.4 typing-inspect-0.8.0 yarl-1.9.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pypdf\n",
            "  Downloading pypdf-3.8.1-py3-none-any.whl (248 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m248.8/248.8 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pypdf\n",
            "Successfully installed pypdf-3.8.1\n"
          ]
        }
      ],
      "source": [
        "!pip install langchain --upgrade\n",
        "# Version: 0.0.164\n",
        "\n",
        "!pip install pypdf"
      ],
      "id": "9d615a77"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2d3e92ed",
        "outputId": "41db9733-4244-428f-a1a7-ecaa0fb587e9"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:langchain.utilities.powerbi:Could not import azure.core python package.\n"
          ]
        }
      ],
      "source": [
        "# PDF Loaders. If unstructured gives you a hard time, try PyPDFLoader\n",
        "from langchain.document_loaders import UnstructuredPDFLoader, OnlinePDFLoader, PyPDFLoader\n",
        "\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "import os"
      ],
      "id": "2d3e92ed"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5166d759"
      },
      "source": [
        "### Load your data"
      ],
      "id": "5166d759"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b4a2d6bf"
      },
      "outputs": [],
      "source": [
        "#loader = PyPDFLoader(\"/content/field-guide-to-data-science.pdf\")\n",
        "\n",
        "## Other options for loaders \n",
        "# loader = UnstructuredPDFLoader(\"/content/ata/field-guide-to-data-science.pdf\")\n",
        "loader = OnlinePDFLoader(\"https://wolfpaulus.com/wp-content/uploads/2017/05/field-guide-to-data-science.pdf\")"
      ],
      "id": "b4a2d6bf"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6NUgaEuIVVbE",
        "outputId": "91476f66-d9df-4c45-93d8-b82d7d8cefa3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting unstructured\n",
            "  Downloading unstructured-0.6.8-py3-none-any.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m26.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting argilla (from unstructured)\n",
            "  Downloading argilla-1.7.0-py3-none-any.whl (2.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m76.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.2)\n",
            "Collecting msg-parser (from unstructured)\n",
            "  Downloading msg_parser-1.2.0-py2.py3-none-any.whl (101 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.8/101.8 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.0.10)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.5.3)\n",
            "Collecting pdfminer.six (from unstructured)\n",
            "  Downloading pdfminer.six-20221105-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m88.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from unstructured) (8.4.0)\n",
            "Collecting pypandoc (from unstructured)\n",
            "  Downloading pypandoc-1.11-py3-none-any.whl (20 kB)\n",
            "Collecting python-docx (from unstructured)\n",
            "  Downloading python-docx-0.8.11.tar.gz (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m75.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-pptx (from unstructured)\n",
            "  Downloading python-pptx-0.6.21.tar.gz (10.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m68.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.4.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.27.1)\n",
            "Requirement already satisfied: certifi>=2022.12.07 in /usr/local/lib/python3.10/dist-packages (from unstructured) (2022.12.7)\n",
            "Collecting httpx<0.24,>=0.15 (from argilla->unstructured)\n",
            "  Downloading httpx-0.23.3-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting deprecated~=1.2.0 (from argilla->unstructured)\n",
            "  Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (23.1)\n",
            "Requirement already satisfied: pydantic>=1.7.1 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.10.7)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.13 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.14.1)\n",
            "Requirement already satisfied: numpy<1.24.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (1.22.4)\n",
            "Requirement already satisfied: tqdm>=4.27.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (4.65.0)\n",
            "Collecting backoff (from argilla->unstructured)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Collecting monotonic (from argilla->unstructured)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting rich<=13.0.1 (from argilla->unstructured)\n",
            "  Downloading rich-13.0.1-py3-none-any.whl (238 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.1/238.1 kB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer<1.0.0,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from argilla->unstructured) (0.7.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->unstructured) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->unstructured) (2022.7.1)\n",
            "Collecting olefile>=0.46 (from msg-parser->unstructured)\n",
            "  Downloading olefile-0.46.zip (112 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2022.10.31)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.10/dist-packages (from openpyxl->unstructured) (1.1.0)\n",
            "Requirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six->unstructured) (2.0.12)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six->unstructured) (40.0.2)\n",
            "Collecting XlsxWriter>=0.5.7 (from python-pptx->unstructured)\n",
            "  Downloading XlsxWriter-3.1.0-py3-none-any.whl (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.7/152.7 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six->unstructured) (1.15.1)\n",
            "Collecting httpcore<0.17.0,>=0.15.0 (from httpx<0.24,>=0.15->argilla->unstructured)\n",
            "  Downloading httpcore-0.16.3-py3-none-any.whl (69 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rfc3986[idna2008]<2,>=1.3 (from httpx<0.24,>=0.15->argilla->unstructured)\n",
            "  Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx<0.24,>=0.15->argilla->unstructured) (1.3.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.7.1->argilla->unstructured) (4.5.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->unstructured) (1.16.0)\n",
            "Collecting commonmark<0.10.0,>=0.9.0 (from rich<=13.0.1->argilla->unstructured)\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.10/dist-packages (from rich<=13.0.1->argilla->unstructured) (2.14.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six->unstructured) (2.21)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore<0.17.0,>=0.15.0->httpx<0.24,>=0.15->argilla->unstructured)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from httpcore<0.17.0,>=0.15.0->httpx<0.24,>=0.15->argilla->unstructured) (3.6.2)\n",
            "Building wheels for collected packages: python-docx, python-pptx, olefile\n",
            "  Building wheel for python-docx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-docx: filename=python_docx-0.8.11-py3-none-any.whl size=184491 sha256=b3885c5bf39bcf462b2d87714b7646f2066782d5aaa5df5b5e511bd132ec11c0\n",
            "  Stored in directory: /root/.cache/pip/wheels/80/27/06/837436d4c3bd989b957a91679966f207bfd71d358d63a8194d\n",
            "  Building wheel for python-pptx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-pptx: filename=python_pptx-0.6.21-py3-none-any.whl size=470935 sha256=ab80e276231f8faa03eccc9e3e19eedd91f966cd808f653b80f5335ab8266254\n",
            "  Stored in directory: /root/.cache/pip/wheels/ea/dd/74/01b3ec7256a0800b99384e9a0f7620e358afc3a51a59bf9b49\n",
            "  Building wheel for olefile (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for olefile: filename=olefile-0.46-py2.py3-none-any.whl size=35417 sha256=5618cbf2dc4afb1bb27281b69074aaf414e4f81c2754a9f91c0eace4386ec4b5\n",
            "  Stored in directory: /root/.cache/pip/wheels/02/39/c0/9eb1f7a42b4b38f6f333b6314d4ed11c46f12a0f7b78194f0d\n",
            "Successfully built python-docx python-pptx olefile\n",
            "Installing collected packages: rfc3986, monotonic, commonmark, XlsxWriter, rich, python-magic, python-docx, pypandoc, olefile, h11, deprecated, backoff, python-pptx, msg-parser, httpcore, pdfminer.six, httpx, argilla, unstructured\n",
            "  Attempting uninstall: rich\n",
            "    Found existing installation: rich 13.3.4\n",
            "    Uninstalling rich-13.3.4:\n",
            "      Successfully uninstalled rich-13.3.4\n",
            "Successfully installed XlsxWriter-3.1.0 argilla-1.7.0 backoff-2.2.1 commonmark-0.9.1 deprecated-1.2.13 h11-0.14.0 httpcore-0.16.3 httpx-0.23.3 monotonic-1.6 msg-parser-1.2.0 olefile-0.46 pdfminer.six-20221105 pypandoc-1.11 python-docx-0.8.11 python-magic-0.4.27 python-pptx-0.6.21 rfc3986-1.5.0 rich-13.0.1 unstructured-0.6.8\n"
          ]
        }
      ],
      "source": [
        "!pip install unstructured"
      ],
      "id": "6NUgaEuIVVbE"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bhywecbsxu0f",
        "outputId": "fff3a4b7-08ed-4c9b-9479-5f18120a1443"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pdf2img\n",
            "  Downloading pdf2img-0.1.2.tar.gz (8.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from pdf2img) (8.4.0)\n",
            "Building wheels for collected packages: pdf2img\n",
            "  Building wheel for pdf2img (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pdf2img: filename=pdf2img-0.1.2-py3-none-any.whl size=9241 sha256=0ad3ff8e34962e8b4ed1aceb36d2acfb932b9f36043549dad84b2051f69d6d3f\n",
            "  Stored in directory: /root/.cache/pip/wheels/7f/ce/db/9cb36b5b9740928098b2a9784653023a09346063988de88907\n",
            "Successfully built pdf2img\n",
            "Installing collected packages: pdf2img\n",
            "Successfully installed pdf2img-0.1.2\n"
          ]
        }
      ],
      "source": [
        "!pip install pdf2img"
      ],
      "id": "bhywecbsxu0f"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bcdac23c",
        "outputId": "a6758adb-bfe9-4f68-d1d1-5a040b6ae386"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        }
      ],
      "source": [
        "data = loader.load()"
      ],
      "id": "bcdac23c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b4fd7c9e",
        "outputId": "b86d1de4-df74-4daf-d416-f78d3c1c7c42"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "You have 1 document(s) in your data\n",
            "There are 72807 characters in your document\n"
          ]
        }
      ],
      "source": [
        "# Note: If you're using PyPDFLoader then it will split by page for you already\n",
        "print (f'You have {len(data)} document(s) in your data')\n",
        "print (f'There are {len(data[0].page_content)} characters in your document')"
      ],
      "id": "b4fd7c9e"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8af9b604"
      },
      "source": [
        "### Chunk your data up into smaller documents"
      ],
      "id": "8af9b604"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fb3c6f02"
      },
      "outputs": [],
      "source": [
        "# Note: If you're using PyPDFLoader then we'll be splitting for the 2nd time.\n",
        "# This is optional, test out on your own data.\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=0)\n",
        "texts = text_splitter.split_documents(data)"
      ],
      "id": "fb3c6f02"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "879873a4",
        "outputId": "eef88796-d5e1-4b0d-aa5e-4cee468d6964"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Now you have 40 documents\n"
          ]
        }
      ],
      "source": [
        "print (f'Now you have {len(texts)} documents')"
      ],
      "id": "879873a4"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BqDMaUjMVxfI",
        "outputId": "f4c7dc0d-c00d-4e3b-ab88-3448ba6a3c45"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[Document(page_content='Everyone you will ever meet knows something you don’t.\\n\\n[1]\\n\\n››\\n\\nT H E S TO RY\\n\\nof T H E F I E L D\\n\\nG U I D E\\n\\nSeveral years ago we created The Field Guide to Data Science because we wanted to help organizations of all types and sizes. There were countless industry and academic publications describing what Data Science is and why we should care, but very little information was available to explain how to make use of data as a resource. We find that situation to be just as true today as we did two years ago, when we created the first edition of the field guide.\\n\\nAt Booz Allen Hamilton, we built an industry-leading team of Data Scientists. Over the course of hundreds of analytic challenges for countless clients, we’ve unraveled the DNA of Data Science. Many people have put forth their thoughts on single aspects of Data Science. We believe we can offer a broad perspective on the conceptual models, tradecraft, processes and culture of Data Science – the what, the why, the who and the how. Companies with strong Data Science teams often focus on a single class of problems – graph algorithms for social network analysis, and recommender models for online shopping are two notable examples. Booz Allen is different. In our role as consultants, we support a diverse set of government and commercial clients across a variety of domains. This allows us to uniquely understand the DNA of Data Science.\\n\\nOur goal in creating The Field Guide to Data Science was to capture what we have learned and to share it broadly. The field of Data Science has continued to advance since we first released the field guide. As a result, we decided to release this second edition, incorporating a few new and important concepts. We also added technical depth and richness that we believe practitioners will find useful.\\n\\nWe want this effort to continue driving forward the science and art of Data Science.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='This field guide came from the passion our team feels for its work. It is not a textbook nor is it a superficial treatment. Senior leaders will walk away with a deeper understanding of the concepts at the heart of Data Science. Practitioners will add to their toolbox. We hope everyone will enjoy the journey.\\n\\n››\\n\\nW E A R E A L L A U T H O R S of T H I S S T O R Y\\n\\nWe recognize that Data Science is a team sport. The Field Guide to Data Science provides Booz Allen Hamilton’s perspective on the complex and sometimes mysterious field of Data Science. We cannot capture all that is Data Science. Nor can we keep up - the pace at which this field progresses outdates work as fast as it is produced. As a result, we opened this field guide to the world as a living document to bend and grow with technology, expertise, and evolving techniques.\\n\\nThank you to all the people that have emailed us your ideas as well as the 100+ people who have watched, starred, or forked our GitHub repository. We truly value the input of the community, as we work together to advance the science and art of Data Science. This is why we have included authors from outside Booz Allen Hamilton on this second edition of The Field Guide to Data Science.\\n\\nIf you find the guide to be useful, neat, or even lacking, then we encourage you to add your expertise, including:\\n\\n›› ›› ›› ››\\n\\nCase studies from which you have learned\\n\\nCitations from journal articles or papers that inspire you\\n\\nAlgorithms and techniques that you love\\n\\nYour thoughts and comments on other people’s additions\\n\\nEmail us your ideas and perspectives at data_science@bah.com or submit them via a pull request on the GitHub repository.\\n\\nJoin our conversation and take the journey with us. Tell us and the world what you know. Become an author of this story.\\n\\n››\\n\\nT H E O U T L I N E of O U R S T O R Y 12 ›› Meet Your Guides 17 ›› The Short Version – The Core Concepts of Data Science 18 ›› Start Here for the Basics – An Introduction to Data Science', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='What Do We Mean by Data Science?\\n\\nHow Does Data Science Actually Work?\\n\\nWhat Does It Take to Create a Data Science Capability?\\n\\n46 ›› Take off the Training Wheels – The Practitioner’s Guide to Data Science\\n\\nGuiding Principles\\n\\nThe Importance of Reason\\n\\nComponent Parts of Data Science\\n\\nFractal Analytic Model\\n\\nThe Analytic Selection Process\\n\\nGuide to Analytic Selection\\n\\nDetailed Table of Analytics\\n\\n84 ›› Life in the Trenches – Navigating Neck Deep in Data\\n\\nGoing Deep into Machine Learning\\n\\nFeature Engineering\\n\\nFeature Selection\\n\\nEnsemble Models\\n\\nData Veracity\\n\\nApplication of Domain Knowledge\\n\\nThe Curse of Dimensionality\\n\\nModel Validation\\n\\n102 ›› Putting it all Together – Our Case Studies\\n\\nStreamlining Medication Review\\n\\nReducing Flight Delays\\n\\nMaking Vaccines Safer\\n\\nForecasting the Relative Risk for the Onset of\\n\\nMass Killings to Help Prevent Future Atrocities\\n\\nPredicting Customer Response\\n\\n114 ›› Closing Time\\n\\nParting Thoughts\\n\\nThe Future of Data Science\\n\\nReferences\\n\\nAbout Booz Allen Hamilton\\n\\n››\\n\\nThe S H O RT V E R S I O N\\n\\n›› Data Science is the art of turning data into actions. It’s all about the tradecraft. Tradecraft is the process, tools and technologies for humans and computers to work together to transform data into insights.\\n\\n›› Data Science tradecraft creates data products.\\n\\nData products provide actionable information without exposing decision makers to the underlying data or analytics (e.g., buy/sell strategies for financial instruments, a set of actions to improve product yield, or steps to improve product marketing).', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='›› Data Science supports and encourages shifting between deductive (hypothesis-based) and inductive (pattern- based) reasoning. This is a fundamental change from traditional analysis approaches. Inductive reasoning and exploratory data analysis provide a means to form or refine hypotheses and discover new analytic paths. Models of reality no longer need to be static. They are constantly tested, updated and improved until better models are found.\\n\\n›› Data Science is necessary for companies to stay with the\\n\\npack and compete in the future. Organizations are constantly making decisions based on gut instinct, loudest voice and best argument – sometimes they are even informed by real information. The winners and the losers in the emerging data economy are going to be determined by their Data Science teams.\\n\\n›› Data Science capabilities can be built over time. Organizations mature through a series of stages – Collect, Describe, Discover, Predict, Advise – as they move from data deluge to full Data Science maturity. At each stage, they can tackle increasingly complex analytic goals with a wider breadth of analytic capabilities. However, organizations need not reach maximum Data Science maturity to achieve success. Significant gains can be found in every stage.\\n\\n›› Data Science is a different kind of team sport.\\n\\nData Science teams need a broad view of the organization. Leaders must be key advocates who meet with stakeholders to ferret out the hardest challenges, locate the data, connect disparate parts of the business, and gain widespread buy-in.\\n\\nThe Short Version\\n\\n17 17\\n\\nWhat do We Mean by Data Science?\\n\\nDescribing Data Science is like trying to describe a sunset – it should be easy, but somehow capturing the words is impossible.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\n» Data Product\\n\\nA data product provides actionable information without exposing decision makers to the underlying data or analytics. Examples include:\\n\\nMovie Recommendations\\n\\nWeather Forecasts', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Stock Market Predictions\\n\\nProduction Process\\n\\nImprovements\\n\\nHealth Diagnosis\\n\\nFlu Trend Predictions\\n\\nTargeted Advertising\\n\\nData Science Defined\\n\\nData Science is the art of turning data into actions. This is accomplished through the creation of data products, which provide actionable information without exposing decision makers to the underlying data or analytics (e.g., buy/sell strategies for financial instruments, a set of actions to improve product yield, or steps to improve product marketing).\\n\\nPerforming Data Science requires the extraction of timely, actionable information from diverse data sources to drive data products. Examples of data products include answers to questions such as: “Which of my products should I advertise more heavily to increase profit? How can I improve my compliance program, while reducing costs? What manufacturing process change will allow me to build a better product?” The key to answering these questions is: understand the data you have and what the data inductively tells you.\\n\\nRead this for additional background:\\n\\nThe term Data Science appeared in the computer science literature throughout the 1960s-1980s. It was not until the late 1990s however, that the field as we describe it here, began to emerge from the statistics and data mining communities (e.g., [2] and [3]). Data Science was first introduced as an independent discipline in 2001.[4] Since that time, there have been countless articles advancing the discipline, culminating with Data Scientist being declared the sexiest job of the 21st century.[5]\\n\\nWe established our first Data Science team at Booz Allen in 2010. It began as a natural extension of our Business Intelligence and cloud', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='infrastructure development work. We saw the need for a new approach to distill value from our clients’ data. We approached the problem with a multidisciplinary team of computer scientists, mathematicians and domain experts. They immediately produced new insights and analysis paths, solidifying the validity of the approach. Since that time, our Data Science team has grown to 250 staff supporting dozens of clients across a variety of domains. This breadth of experience provides a unique perspective on the conceptual models, tradecraft, processes and culture of Data Science.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n21 21\\n\\nWhat is the Impact of Data Science?\\n\\nAs we move into the data economy, Data Science is the competitive advantage for organizations interested in winning – in whatever way winning is defined. The manner in which the advantage is defined is through improved decision-making. A former colleague liked to describe data-informed decision making like this: If you have perfect information or zero information then your task is easy – it is in between those two extremes that the trouble begins. What he was highlighting is the stark reality that whether or not information is available, decisions must be made.\\n\\nThe way organizations make decisions has been evolving for half a century. Before the introduction of Business Intelligence, the only options were gut instinct, loudest voice, and best argument. Sadly, this method still exists today, and in some pockets it is the predominant means by which the organization acts. Take our advice and never, ever work for such a company!', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Fortunately for our economy, most organizations began to inform their decisions with real information through the application of simple statistics. Those that did it well were rewarded; those that did not failed. We are outgrowing the ability of simple stats to keep pace with market demands, however. The rapid expansion of available data and the tools to access and make use of the data at scale are enabling fundamental changes to the way organizations make decisions.\\n\\nData Science is required to maintain competitiveness in the increasingly data-rich environment. Much like the application of simple statistics, organizations that embrace Data Science will be rewarded while those that do not will be challenged to keep pace. As more complex, disparate datasets become available, the chasm between these groups will only continue to widen. The figure, The Business Impacts of Data Science, highlights the value awaiting organizations that embrace Data Science.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nWhat is Different Now?\\n\\nFor 20 years IT systems were built the same way. We separated the people who ran the business from the people who managed the infrastructure (and therefore saw data as simply another thing they had to manage). With the advent of new technologies and analytic techniques, this artificial – and highly ineffective – separation of critical skills is no longer necessary. For the first time, organizations can directly connect business decision makers to the data. This simple step transforms data from being ‘something to be managed’ into ‘something to be valued.’\\n\\nIn the wake of the transformation, organizations face a stark choice: you can continue to build data silos and piece together disparate information or you can consolidate your data and distill answers.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='From the Data Science perspective, this is a false choice: The siloed approach is untenable when you consider the (a) the opportunity cost of not making maximum use of all available data to help an organization succeed, and (b) the resource and time costs of continuing down the same path with outdated processes. The tangible benefits of data products include:\\n\\n››\\n\\n››\\n\\nOpportunity Costs: Because Data Science is an emerging field, opportunity costs arise when a competitor implements and generates value from data before you. Failure to learn and account for changing customer demands will inevitably drive customers away from your current offerings. When competitors are able to successfully leverage Data Science to gain insights, they can drive differentiated customer value propositions and lead their industries as a result.\\n\\nEnhanced Processes: As a result of the increasingly interconnected world, huge amounts of data are being generated and stored every instant. Data Science can be used to transform data into insights that help improve existing processes. Operating costs can be driven down dramatically by effectively incorporating the complex interrelationships in data like never before. This results in better quality assurance, higher product yield and more effective operations.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n27 27\\n\\nHow does Data Science Actually Work?\\n\\nIt’s not rocket science… it’s something better - Data Science\\n\\nLet’s not kid ourselves - Data Science is a complex field. It is difficult, intellectually taxing work, which requires the sophisticated integration of talent, tools and techniques. But as a field guide, we need to cut through the complexity and provide a clear, yet effective way to understand this new world.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='To do this, we will transform the field of Data Science into a set of simplified activities as shown in the figure, The Four Key Activities of a Data Science Endeavor. Data Science purists will likely disagree with this approach, but then again, they probably don’t need a field guide, sitting as they do in their ivory towers! In the real world, we need clear and simple operating models to help drive us forward.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nAcquire\\n\\nAll analysis starts with access to data, and for the Data Scientist this axiom holds true. But there are some significant differences – particularly with respect to the question of who stores, maintains and owns the data in an organization.\\n\\nBut before we go there, lets look at what is changing. Traditionally, rigid data silos artificially define the data to be acquired. Stated another way, the silos create a filter that lets in a very small amount of data and ignores the rest. These filtered processes give us an artificial view of the world based on the ‘surviving data,’ rather than one that shows full reality and meaning. Without a broad and expansive dataset, we can never immerse ourselves in the diversity of the data. We instead make decisions based on limited and constrained information.\\n\\nEliminating the need for silos gives us access to all the data at once – including data from multiple outside sources. It embraces the reality that diversity is good and complexity is okay. This mindset creates a completely different way of thinking about data in an organization by giving it a new and differentiated role. Data represents a significant new profit and mission-enhancement opportunity for organizations.\\n\\nBut as mentioned earlier, this first activity is heavily dependent upon the situation and circumstances. We can’t leave you with anything more than general guidance to help ensure maximum value:\\n\\n››\\n\\n››\\n\\n››\\n\\n››', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Look inside first: What data do you have current access to that you are not using? This is in large part the data being left behind by the filtering process, and may be incredibly valuable.\\n\\nRemove the format constraints: Stop limiting your data acquisition mindset to the realm of structured databases. Instead, think about unstructured and semi-structured data as viable sources.\\n\\nFigure out what’s missing: Ask yourself what data would make a big difference to your processes if you had access to it. Then go find it!\\n\\nEmbrace diversity: Try to engage and connect to publicly available sources of data that may have relevance to your domain area.\\n\\n» Not All Data Is Created Equal\\n\\nAs you begin to aggregate data, remember that not all data is created equally. Organizations have a tendency to collect any data that is available. Data that is nearby (readily accessible and easily obtained) may be cheap to collect, but there is no guarantee it is the right data to collect. Focus on the data with the highest ROI for your organization. Your Data Science team can help identify that data. Also remember that you need to strike a balance between the data that you need and the data that you have. Collecting huge volumes of data is useless and costly if it is not the data that you need.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nPrepare\\n\\nOnce you have the data, you need to prepare it for analysis.\\n\\nOrganizations often make decisions based on inexact data. Data stovepipes mean that organizations may have blind spots. They are not able to see the whole picture and fail to look at their data and challenges holistically. The end result is that valuable information is withheld from decision makers. Research has shown almost 33% of decisions are made without good data or information. [10]', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='When Data Scientists are able to explore and analyze all the data, new opportunities arise for analysis and data-driven decision making. The insights gained from these new opportunities will significantly change the course of action and decisions within an organization. Gaining access to an organization’s complete repository of data, however, requires preparation.\\n\\nOur experience shows time and time again that the best tool for Data Scientists to prepare for analysis is a lake – specifically, the Data Lake.[11] This is a new approach to collecting, storing and integrating data that helps organizations maximize the utility of their data. Instead of storing information in discrete data structures, the Data Lake consolidates an organization’s complete repository of data in a single, large view. It eliminates the expensive and cumbersome data-preparation process, known as Extract/Transform/Load (ETL), necessary with data silos. The entire body of information in the Data Lake is available for every inquiry – and all at once.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n31 31\\n\\nData Scientists work across the spectrum of analytic goals – Describe, Discover, Predict and Advise. The maturity of an analytic capability determines the analytic goals encompassed. Many variables play key roles in determining the difficulty and suitability of each goal for an organization. Some of these variables are the size and budget of an organization and the type of data products needed by the decision makers. A detailed discussion on analytic maturity can be found in Data Science Maturity within an Organization.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='In addition to consuming the greatest effort, the Analyze activity is by far the most complex. The tradecraft of Data Science is an art. While we cannot teach you how to be an artist, we can share foundational tools and techniques that can help you be successful. The entirety of Take Off the Training Wheels is dedicated to sharing insights we have learned over time while serving countless clients. This includes descriptions of a Data Science product lifecycle and the Fractal Analytic Model (FAM). The Analytic Selection Process and accompanying Guide to Analytic Selection provide key insights into one of the most challenging tasks in all of Data Science – selecting the right technique for the job.\\n\\nAct\\n\\nNow that we have analyzed the data, it’s time to take action.\\n\\nThe ability to make use of the analysis is critical. It is also very situational. Like the Acquire activity, the best we can hope for is to provide some guiding principles to help you frame the output for maximum impact. Here are some key points to keep in mind when presenting your results:\\n\\n1. The finding must make sense with relatively little up-front\\n\\ntraining or preparation on the part of the decision maker.\\n\\n2. The finding must make the most meaningful patterns, trends\\n\\nand exceptions easy to see and interpret.\\n\\n3. Every effort must be made to encode quantitative data\\n\\naccurately so the decision maker can accurately interpret and compare the data.\\n\\n4. The logic used to arrive at the finding must be clear and compelling as well as traceable back through the data.\\n\\n5. The findings must answer real business questions.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n33 33\\n\\nWhen organizations start out, they have Data Silos. At this stage, they have not carried out any broad Aggregate activities. They may not have a sense of all the data they have or the data they need. The decision to create a Data Science capability signals the transition into the Collect stage.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='All of your initial effort will be focused on identifying and aggregating data. Over time, you will have the data you need and a smaller proportion of your effort can focus on Collect. You can now begin to Describe your data. Note, however, that while the proportion of time spent on Collect goes down dramatically, it never goes away entirely. This is indicative of the four activities outlined earlier – you will continue to Aggregate and Prepare data as new analytic questions arise, additional data is needed and new data sources become available.\\n\\nOrganizations continue to advance in maturity as they move through the stages from Describe to Advise. At each stage they can tackle increasingly complex analytic goals with a wider breadth of analytic capabilities. As described for Collect, each stage never goes away entirely. Instead, the proportion of time spent focused on it goes down and new, more mature activities begin. A brief description of each stage of maturity is shown in the table The Stages of Data Science Maturity.\\n\\nThe Stages of Data Science Maturity\\n\\nStage\\n\\nDescription\\n\\nExample\\n\\nCollect\\n\\nFocuses on collecting internal or external datasets.\\n\\nGathering sales records and corresponding weather data.\\n\\nDescribe\\n\\nSeeks to enhance or reﬁne raw data as well as leverage basic analytic functions such as counts.\\n\\nHow are my customers distributed with respect to location, namely zip code?\\n\\nDiscover\\n\\nIdentiﬁes hidden relationships or patterns.\\n\\nAre there groups within my regular customers that purchase similarly?\\n\\nPredict\\n\\nUtilizes past observations to predict future observations.\\n\\nCan we predict which products that certain customer groups are more likely to purchase?\\n\\nAdvise\\n\\nDeﬁnes your possible decisions, optimizes over those decisions, and advises to use the decision that gives the best outcome.\\n\\nYour advice is to target advertise to speciﬁc groups for certain products to maximize revenue.\\n\\nSource: Booz Allen Hamilton\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='The maturity model provides a powerful tool for understanding and appreciating the maturity of a Data Science capability. Organizations need not reach maximum maturity to achieve success. Significant gains can be found in every stage. We believe strongly that one does not engage in a Data Science effort, however, unless it is intended to produce an output – that is, you have the intent to Advise. This means simply that each step forward in maturity drives you to the right in the model diagram. Moving to the right requires the correct processes, people, culture and operating model – a robust Data Science capability. What Does it Take to Create a Data Science Capability? addresses this topic.\\n\\nWe have observed very few organizations actually operating at the highest levels of maturity, the Predict and Advise stages. The tradecraft of Discover is only now maturing to the point that organizations can focus on advanced Predict and Advise activities. This is the new frontier of Data Science. This is the space in which we will begin to understand how to close the cognitive gap between humans and computers. Organizations that reach Advise will be met with true insights and real competitive advantage.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n37 37\\n\\n» Where does your organization\\n\\nfall in analytic maturity?\\n\\nTake the quiz!\\n\\n1. How many data sources do\\n\\nyou collect?\\n\\na. Why do we need a bunch of data?\\n\\n– 0 points, end here.\\n\\nb. I don’t know the exact number.\\n\\n– 5 points\\n\\nc. We identiﬁed the required data and\\n\\ncollect it. – 10 points\\n\\n2. Do you know what questions\\n\\nyour Data Science team is trying to answer?\\n\\na. Why do we need questions?\\n\\n0 points\\n\\nb. No, they ﬁgure it out for themselves.\\n\\n– 5 points\\n\\nc. Yes, we evaluated the questions that will have the largest impact to the business. – 10 points\\n\\n3. Do you know the important factors\\n\\ndriving your business?\\n\\na. I have no idea. – 0 points\\n\\nb. Our quants help me ﬁgure it out.\\n\\n– 5 points', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='c. We have a data product for that.\\n\\n– 10 points\\n\\n4. Do you have an understanding of\\n\\nfuture conditions?\\n\\na. I look at the current conditions and\\n\\nread the tea leaves. – 0 points\\n\\nb. We have a data product for that.\\n\\n– 5 points\\n\\n5. Do you know the best course of action to take for your key decisions?\\n\\na. I look at the projections and plan a\\n\\ncourse. – 0 points\\n\\nb. We have a data product for that.\\n\\n– 5 points\\n\\nCheck your score:\\n\\n0 – Data Silos, 5-10 – Collect, 10-20 – Describe, 20-30 – Discover, 30-35 – Predict, 35-40 - Advise\\n\\nSource: Booz Allen Hamilton\\n\\n» The Triple Threat Unicorn\\n\\nIndividuals who are great at all three of the Data Science foundational technical skills are like unicorns – very rare and if you’re ever lucky enough to find one they should be treated carefully. When you manage these people:\\n\\n› Encourage them to lead your team, but not manage it. Don’t bog them down with responsibilities of management that could be done by other staff.\\n\\n› Put extra effort into managing\\n\\ntheir careers and interests within your organization. Build opportunities for promotion into your organization that allow them to focus on mentoring other Data Scientists and progressing the state of the art while also advancing their careers.\\n\\n› Make sure that they have the opportunity to present and spread their ideas in many different forums, but also be sensitive to their time.\\n\\nUnderstanding What Makes a Data Scientist\\n\\nData Science often requires a significant investment of time across a variety of tasks. Hypotheses must be generated and data must be acquired, prepared, analyzed, and acted upon. Multiple techniques are often applied before one yields interesting results. If that seems daunting, it is because it is. Data Science is difficult, intellectually taxing work, which requires lots of talent: both tangible technical skills as well as the intangible “x-factors.”', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='There are four independent yet comprehensive foundational Data Science competency clusters that, when considered together, convey the essence of what it means to be a successful Data Scientist. There are also reach back competencies that complement the foundational clusters but do not define the core tradecraft or attributes of the Data Science team.\\n\\nData Science Competency Framework (see [13] for complete framework)\\n\\nClusters\\n\\nCompetencies\\n\\nDescription\\n\\nTechnical: “Knows How and What to do”\\n\\nAdvanced Mathematics; Computer Science; Data Mining and Integration; Database Science; Research Design; Statistical Modeling; Machine Learning; Operations Research; Programming and Scripting\\n\\nThe technical competency cluster depicts the foundational technical and specialty knowledge and skills needed for successful performance in each job or role.\\n\\nData Science Consulting: “Can Do in a Client and Customer Environment”\\n\\nCollaboration and Teamwork; Communications; Data Science Consulting; Ethics and Integrity\\n\\nCognitive: “Able to Do or Learn to Do”\\n\\nCritical Thinking; Inductive and Deductive Reasoning; Problem Solving\\n\\nThe characteristics in the consulting competency cluster can help Data Scientists easily integrate into various market or domain contexts and partner with business units to understand the environment and solve complex problems.\\n\\nThe cognitive competency cluster represents the type of critical thinking and reasoning abilities (both inductive and deductive) a Data Scientist should have to perform their job.\\n\\nPersonality: “Willing or Motivated to Do”\\n\\nAdaptability/Flexibility; Ambiguity Tolerance; Detail Orientation; Innovation and Creativity; Inquisitiveness; Perseverance; Resilience and Hardiness; Self- Conﬁdence; Work Ethic\\n\\nThe personality competency cluster describes the personality traits that drive behaviors that are beneﬁcial to Data Scientists, such as inquisitiveness, creativity, and perseverance.\\n\\nReach Back Competencies for Data Science Teams', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Business Acumen; Data Visualization; Domain Expertise; Program Management\\n\\nSource: Booz Allen Hamilton\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n41 41\\n\\nThe most important qualities of Data Scientists tend to be the intangible aspects of their personalities. Data Scientists are by nature curious, creative, focused, and detail-oriented.\\n\\n››\\n\\n››\\n\\n››\\n\\n››\\n\\nCuriosity is necessary to peel apart a problem and examine the interrelationships between data that may appear superficially unrelated.\\n\\nCreativity is required to invent and try new approaches to solving a problem, which often times have never been applied in such a context before.\\n\\nFocus is required to design and test a technique over days and weeks, find it doesn’t work, learn from the failure, and try again.\\n\\nAttention to Detail is needed to maintain rigor, and to detect and avoid over-reliance on intuition when examining data.\\n\\nWe have found the single most important attribute is flexibility in overcoming setbacks - the willingness to abandon one idea and try a new approach. Often, Data Science is a series of dead ends before, at last, the way forward is identified. It requires a unique set of personality attributes to succeed in such an environment. Technical skills can be developed over time: the ability to be flexible – and patient, and persistent – cannot.\\n\\nFinding the Athletes for Your Team', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Building a Data Science team is complex. Organizations must simultaneously engage existing internal staff to create an “anchor” who can be used to recruit and grow the team, while at the same time undergo organizational change and transformation to meaningfully incorporate this new class of employee. Building a team starts with identifying existing staff within an organization who have a high aptitude for Data Science. Good candidates will have a formal background in any of the three foundational technical skills we mentioned, and will most importantly have the personality traits necessary for Data Science. They may often have advanced (masters or higher) degrees, but not always. The very first staff you identify should also have good leadership traits and a sense of purpose for the organization, as they will lead subsequent staffing and recruiting efforts. Don’t discount anyone – you will find Data Scientists in the strangest places with the oddest combinations of backgrounds.\\n\\n» Don’t judge a book by its cover, or a Data Scientist by his or her degree in this case. Amazing Data Scientists can be found anywhere. Just look at the diverse and surprising sampling of degrees held by Our Experts:\\n\\n› Bioinformatics\\n\\n› Biomedical Engineering\\n\\n› Biophysics\\n\\n› Business\\n\\n› Computer Graphics\\n\\n› Computer Science\\n\\n› English\\n\\n› Forest Management\\n\\n› History\\n\\n›\\n\\nIndustrial Engineering\\n\\n›\\n\\nInformation Technology\\n\\n› Mathematics\\n\\n› National Security Studies\\n\\n› Operations Research\\n\\n› Physics\\n\\n› Wildlife & Fisheries Management\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nShaping the Culture\\n\\nIt is no surprise—building a culture is hard and there is just as much art to it as there is science. It is about deliberately creating the conditions for Data Science to flourish (for both Data Scientists and the average employee). You can then step back to empower collective ownership of an organic transformation.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content=\"Data Scientists are fundamentally curious and imaginative. We have a saying on our team, “We’re not nosy, we’re Data Scientists.” These qualities are fundamental to the success of the project and to gaining new dimensions on challenges and questions. Often Data Science projects are hampered by the lack of the ability to imagine something new and different. Fundamentally, organizations must foster trust and transparent communication across all levels, instead of deference to authority, in order to establish a strong Data Science team. Managers should be prepared to invite participation more frequently, and offer explanation or apology less frequently.\\n\\nIt is important to provide a path into the Data Science “club” and to empower the average employee to feel comfortable and conversant with Data Science. For something to be part of organizational culture, it must be part of the fabric of the employee behavior. That means employees must interact with and use data products in their daily routines. Another key ingredient to shaping the right culture is that all employees need a baseline of Data Science knowledge, starting with a common lexicon, to facilitate productive collaboration and instill confidence. While not everyone will be Data Scientists, employees need to identify with Data Science and be equipped with the knowledge, skills, and abilities to work with Data Scientists to drive smarter decisions and deliver exponential organizational performance.\\n\\n» “I'm not nosey, I'm a Data\\n\\nScientist”\\n\\n› Always remember that\\n\\nunrelenting curiosity and imagination should be the hallmarks of Data Science. They are fundamental to the success of every Data Science project.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n43 43\\n\\nHow to Generate Momentum\", metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='A Data Science effort can start at the grass roots level by a few folks tackling hard problems, or as directed by the Chief Executive Officer, Chief Data Officer, or Chief Analytics Officer. Regardless of how an effort starts, political headwinds often present more of a challenge than solving any technical hurdles. To help battle the headwinds, it is important to generate momentum and prove the value a Data Science team can provide. The best way to achieve this is usually through a Data Science prototype or proof of concept. Proofs of concepts can generate the critical momentum needed to jump start any Data Science Capability Four qualities, in particular, are essential for every Data Science prototype:\\n\\n1. Organizational Buy-in: A prototype will only succeed if the individuals involved believe in it and are willing to do what they can to make it successful. A good way to gauge interest is to meet with the middle managers; their views are usually indicative of the larger group.\\n\\n2. Clear ROI: Before choosing a prototype problem, ensure that the ROI of the analytic output can be clearly and convincingly demonstrated for both the project and the organization as a whole. This outcome typically requires first reaching consensus on how the ROI will be determined and measured, so that the benefit can be quantified.\\n\\n3. Necessary Data: Before selecting a prototype, you must first\\n\\ndetermine exactly what data is needed, whether it will actually be available, and what it will cost in terms of time and expense. It is important to note that organizations do not need all the possible data – they can still create successful analytics even with some gaps.\\n\\n4. Limited Complexity and Duration: The problem addressed', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='by the prototype should achieve a balance between being too complex and too easy. Organizations new to Data Science often try to show its value with highly complex projects. However, the greater the complexity, the greater the risk of failure. At the same time, if the problem is too easy to solve, senior leaders and others in the organization may not see the need for Data Science. Look for efforts that could benefit from large datasets, or bringing together disparate datasets that have never been combined before, as opposed to those that require complex analytic approaches. In these cases, there is often low-hanging fruit that can lead to significant value for the organization.\\n\\nStart Here for the Basics Start Here for the Basics\\n\\n45 45\\n\\nTA K E O F F the T R A I N I N G W H E E L S\\n\\nTHE PRACTITIONER’S GUIDE TO DATA SCIENCE\\n\\nRead this section to get beyond the hype and learn the secrets of being a Data Scientist.\\n\\nGuiding Principles\\n\\nFailing is good; failing quickly is even better.\\n\\nThe set of guiding principles that govern how we conduct the tradecraft of Data Science are based loosely on the central tenets of innovation, as the two areas are highly connected. These principles are not hard and fast rules to strictly follow, but rather key tenets that have emerged in our collective consciousness. You should use these to guide your decisions, from problem decomposition through implementation.\\n\\n› Be willing to fail. At the core of Data Science is the idea of', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='experimentation. Truly innovative solutions only emerge when you experiment with new ideas and applications. Failure is an acceptable byproduct of experimentation. Failures locate regions that no longer need to be considered as you search for a solution. › Fail often and learn quickly. In addition to a willingness to fail, be ready to fail repeatedly. There are times when a dozen approaches must be explored in order to find the one that works. While you shouldn’t be concerned with failing, you should strive to learn from the attempt quickly. The only way you can explore a large number of solutions is to do so quickly.\\n\\n› Keep the goal in mind. You can often get lost in the details and challenges of an implementation. When this happens, you lose sight of your goal and begin to drift off the path from data to analytic action. Periodically step back, contemplate your goal, and evaluate whether your current approach can really lead you where you want to go.\\n\\n› Dedication and focus lead to success. You must often explore\\n\\nmany approaches before finding the one that works. It’s easy to become discouraged. You must remain dedicated to your analytic goal. Focus on the details and the insights revealed by the data. Sometimes seemingly small observations lead to big successes.\\n\\n» Tips From the Pros\\n\\nIt can be easier to rule out a solution\\n\\nthan conﬁrm its correctness. As a\\n\\nresult, focus on exploring obvious\\n\\nshortcomings that can quickly\\n\\ndisqualify an approach. This will allow\\n\\nyou to focus your time on exploring\\n\\ntruly viable approaches as opposed to\\n\\ndead ends.\\n\\n» Tips From the Pros\\n\\nIf the ﬁrst thing you try to do is to\\n\\n› Complicated does not equal better. As technical practitioners, we\\n\\ncreate the ultimate solution, you will\\n\\nfail, but only after banging your head\\n\\nagainst a wall for several weeks.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='have a tendency to explore highly complex, advanced approaches. While there are times where this is necessary, a simpler approach can often provide the same insight. Simpler means easier and faster to prototype, implement and verify.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nThe Importance of Reason\\n\\nBeware: in the world of Data Science, if it walks like a duck and quacks like a duck, it might just be a moose.\\n\\nData Science supports and encourages shifting between deductive (hypothesis-based) and inductive (pattern-based) reasoning. Inductive reasoning and exploratory data analysis provide a means to form or refine hypotheses and discover new analytic paths. Models of reality no longer need to be static. They are constantly tested, updated and improved until better models are found.\\n\\nThe analysis of big data has brought inductive reasoning to the forefront. Massive amounts of data are analyzed to identify correlations. However, a common pitfall to this approach is confusing correlation with causation. Correlation implies but does not prove causation. Conclusions cannot be drawn from correlations until the underlying mechanisms that relate the data elements are understood. Without a suitable model relating the data, a correlation may simply be a coincidence.\\n\\n» Correlation without\\n\\nCausation\\n\\nA common example of this phenomenon is the high correlation between ice cream consumption and the murder rate during the summer months. Does this mean ice cream consumption causes murder or, conversely, murder causes ice cream consumption? Most likely not, but you can see the danger in mistaking correlation for causation. Our job as Data Scientists is making sure we understand the difference.\\n\\nTake off the Training Wheels\\n\\n49\\n\\nComponent Parts of Data Science\\n\\nThere is a web of components that interact to create your solution space. Understanding how they are connected is critical to your ability to engineer solutions to Data Science problems.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='The components involved in any Data Science project fall into a number of different categories including the data types analyzed, the analytic classes used, the learning models employed and the execution models used to run the analytics. The interconnection across these components, shown in the figure, Interconnection Among the Component Parts of Data Science, speaks to the complexity of engineering Data Science solutions. A choice made for one component exerts influence over choices made for others categories. For example, data types lead the choices in analytic class and learning models, while latency, timeliness and algorithmic parallelization strategy inform the execution model. As we dive deeper into the technical aspects of Data Science, we will begin with an exploration of these components and touch on examples of each.\\n\\nRead this to get the quick and dirty:\\n\\nWhen engineering a Data Science solution, work from an understanding of the components that define the solution space. Regardless of your analytic goal, you must consider the data types with which you will be working, the classes of analytics you will use to generate your data product,\\n\\nhow the learning models embodied will operate and evolve, and the execution models that will govern how the analytic will be run. You will be able to articulate a complete Data Science solution only after considering each of these aspects.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nData Types', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Data types and analytic goals go hand-in-hand much like the chicken and the egg; it is not always clear which comes first. Analytic goals are derived from business objectives, but the data type also influences the goals. For example, the business objective of understanding consumer product perception drives the analytic goal of sentiment analysis. Similarly, the goal of sentiment analysis drives the selection of a text-like data type such as social media content. Data type also drives many other choices when engineering your solutions.\\n\\nThere are a number of ways to classify data. It is common to characterize data as structured or unstructured. Structured data exists when information is clearly broken out into fields that have an explicit meaning and are highly categorical, ordinal or numeric. A related category, semi-structured, is sometimes used to describe structured data that does not conform to the formal structure of data models associated with relational databases or other forms of data tables, but nonetheless contains tags or other markers. Unstructured data, such as natural language text, has less clearly delineated meaning. Still images, video and audio often fall under the category of unstructured data. Data in this form requires preprocessing to identify and extract relevant ‘features.’ The features are structured information that are used for indexing and retrieval, or training classification, or clustering models.\\n\\nData may also be classified by the rate at which it is generated, collected or processed. The distinction is drawn between streaming data that arrives constantly like a torrent of water from a fire hose, and batch data, which arrives in buckets. While there is rarely a connection between data type and data rate, data rate has significant influence over the execution model chosen for analytic implementation and may also inform a decision of analytic class or learning model.\\n\\nTake off the Training Wheels\\n\\n55\\n\\nFractal Analytic Model', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Data Science analytics are a lot like broccoli.\\n\\nFractals are mathematical sets that display self-similar patterns. As you zoom in on a fractal, the same patterns reappear. Imagine a stalk of broccoli. Rip off a piece of broccoli and the piece looks much like the original stalk. Progressively smaller pieces of broccoli still look like the original stalk.\\n\\nData Science analytics are a lot like broccoli – fractal in nature in both time and construction. Early versions of an analytic follow the same development process as later versions. At any given iteration, the analytic itself is a collection of smaller analytics that often decompose into yet smaller analytics.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nGOAL\\n\\nYou must first have some idea of your analytic goal and the end state of the analysis. Is it to Discover, Describe, Predict, or Advise? It is probably a combination of several of those. Be sure that before you start, you define the business value of the data and how you plan to use the insights to drive decisions, or risk ending up with interesting but non-actionable trivia.\\n\\nDATA\\n\\nData dictates the potential insights that analytics can provide. Data Science is about finding patterns in variable data and comparing those patterns. If the data is not representative of the universe of events you wish to analyze, you will want to collect that data through carefully planned variations in events or processes through A/B testing or design of experiments. Datasets are never perfect so don’t wait for perfect data to get started. A good Data Scientist is adept at handling messy data with missing or erroneous values. Just make sure to spend the time upfront to clean the data or risk generating garbage results.\\n\\nCOMPUTATION', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Computation aligns the data to goals through the process of creating insights. Through divide and conquer, computation decomposes into several smaller analytic capabilities with their own goals, data, computation and resulting actions, just like a smaller piece of broccoli maintains the structure of the original stalk. In this way, computation itself is fractal. Capability building blocks may utilize different types of execution models such as batch computation or streaming, that individually accomplish small tasks. When properly combined together, the small tasks produce complex, actionable results.\\n\\nACTION\\n\\nHow should engineers change the manufacturing process to generate higher product yield? How should an insurance company choose which policies to offer to whom and at what price? The output of computation should enable actions that align to the goals of the data product. Results that do not support or inspire action are nothing but interesting trivia.\\n\\nGiven the fractal nature of Data Science analytics in time and construction, there are many opportunities to choose fantastic or shoddy analytic building blocks. The Analytic Selection Process offers some guidance.\\n\\nTake off the Training Wheels\\n\\n63\\n\\nThe Analytic Selection Process\\n\\nIf you focus only on the science aspect of Data Science you will never become a data artist.\\n\\nA critical step in Data Science is to identify an analytic technique that will produce the desired action. Sometimes it is clear; a characteristic of the problem (e.g., data type) points to the technique you should implement. Other times, however, it can be difficult to know where to begin. The universe of possible analytic techniques is large. Finding your way through this universe is an art that must be practiced. We are going to guide you on the next portion of your journey - becoming a data artist.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nTechnique\\n\\nDescription\\n\\nTips From the Pros\\n\\nReferences we love to read', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content=\"Collaborative Filtering\\n\\nAlso known as 'Recommendation,' suggest or eliminate items from a set by comparing a history of actions against items performed by users. Finds similar items based on who used them or similar users based on the items they use.\\n\\nCoordinate Transforma\\n\\n\\n\\ntion\\n\\nProvides a different perspective on data.\\n\\nDeep Learning\\n\\nMethod that learns features that leads to higher concept learning. Usually very deep neural network architectures.\\n\\nUse Singular Value Decomposition based Recommendation for cases where there are latent factors in your domain, e.g., genres in movies.\\n\\nOwen, Sean, Robin Anil, Ted Dunning, and Ellen Friedman. Mahout in Action. New Jersey: Manning, 2012. Print.\\n\\nChanging the coordinate system for data, for example, using polar or cylindrical coordinates, may more readily highlight key structure in the data. A key step in coordinate transformations is to appreciate multidimensionality and to systematically analyze subspaces of the data.\\n\\nAbbott, Edwin A., Flatland: A Romance of Many Dimensions. United Kingdom: Seely & Co., 1884. Print.\\n\\nUtilize a GPU to efﬁciently train complex models.\\n\\nBengio, Yoshua, and Yann LeCun. “Scaling Learning Algorithms towards AI.” Large- Scale Kernel Machines. New York: MIT Press, 2007. Print.\\n\\nMontgomery, Douglas. Design and Analysis of Experiments. New Jersey: John Wiley & Sons, 2012. Print.\\n\\nDesign of Experiments\\n\\nApplies controlled experiments to quantify effects on system output caused by changes to inputs.\\n\\nFractional factorial designs can signiﬁcantly reduce the number of different types of experiments you must conduct.\\n\\nDifferential Equations\\n\\nUsed to express relationships between functions and their derivatives, for example, change over time.\\n\\nDifferential equations can be used to formalize models and make predictions. The equations themselves can be solved numerically and tested with different initial conditions to study system trajectories.\", metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Zill, Dennis, Warren Wright, and Michael Cullen. Differential Equations with Boundary-Value Problems. Connecticut: Cengage Learning, 2012. Print.\\n\\nDiscrete Event Simulation\\n\\nSimulates a discrete sequence of events where each event occurs at a particular instant in time. The model updates its state only at points in time when events occur.\\n\\nDiscrete event simulation is useful when analyzing event based processes such as production lines and service centers to determine how system level behavior changes as different process parameters change. Optimization can integrate with simulation to gain efﬁciencies in a process.\\n\\nBurrus, C. Sidney, Ramesh A. Gopinath, Haitao Guo, Jan E. Odegard and Ivan W. Selesnick. Introduction to Wavelets and Wavelet Transforms: A Primer. New Jersey: Prentice Hall, 1998. Print.\\n\\nDiscrete Wavelet Transform\\n\\nTransforms time series data into frequency domain preserving locality information.\\n\\nOffers very good time and frequency localization. The advantage over Fourier transforms is that it preserves both frequency and locality.\\n\\nEnsemble Learning\\n\\nLearning multiple models and combining output to achieve better performance.\\n\\nBe careful not to overﬁt data by having too many model parameters and overtraining.\\n\\nExpert Systems\\n\\nSystems that use symbolic logic to reason about facts. Emulates human reasoning.\\n\\nUseful to have a human readable explanation of why a system came to a conclusion.\\n\\nExponential Smoothing\\n\\nUsed to remove artifacts expected from collection error or outliers.\\n\\nIn comparison to a using moving average where past observations are weighted equally, exponential smoothing assigns exponentially decreasing weights over time.\\n\\nBurrus, C.Sidney, Ramesh A. Gopinath, Haitao Guo, Jan E. Odegard, and Ivan W. Selesnick. Introduction to Wavelets and Wavelet Transforms: A Primer. New Jersey: Prentice Hall, 1998. Print.\\n\\nDietterich, Thomas G. “Ensemble Methods in Machine Learning.” Lecture Notes in Computer Science. Springer, 2000. Print.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Shortliffe, Edward H., and Bruce G. Buchanan. “A Model of Inexact Reasoning in Medicine.” Mathematical Biosciences. Elsevier B.V., 1975. Print.\\n\\nChatﬁeld, Chris, Anne B. Koehler, J. Keith Ord, and Ralph D. Snyder. “A New Look at Models for Exponential Smoothing.” Journal of the Royal Statistical Society: Series D (The Statistician). Royal Statistical Society, 2001. Print.\\n\\nCompiled by: Booz Allen Hamilton\\n\\nTake off the Training Wheels\\n\\n79\\n\\nTechnique\\n\\nDescription\\n\\nTips From the Pros\\n\\nReferences we love to read\\n\\nFactor Analysis\\n\\nFast Fourier Transform\\n\\nFormat Conversion\\n\\nDescribes variability among correlated variables with the goal of lowering the number of unobserved variables, namely, the factors.\\n\\nTransforms time series from time to frequency domain efﬁciently. Can also be used for image improvement by spatial transforms.\\n\\nCreates a standard representation of data regardless of source format. For example, extracting raw UTF-8 encoded text from binary ﬁle formats such as Microsoft Word or PDFs.\\n\\nIf you suspect there are inmeasurable inﬂuences on your data, then you may want to try factor analysis.\\n\\nChild, Dennis. The Essentials of Factor Analysis. United Kingdom: Cassell Educational, 1990. Print.\\n\\nFiltering a time varying signal can be done more effectively in the frequency domain. Also, noise can often be identiﬁed in such signals by observing power at aberrant frequencies.\\n\\nMitra, Partha P., and Hemant Bokil. Observed Brain Dynamics. United Kingdom: Oxford University Press, 2008. Print.\\n\\nThere are a number of open source software packages that support format conversion and can interpret a wide variety of formats. One notable package is Apache Tikia.\\n\\nIngersoll, Grant S., Thomas S. Morton, and Andrew L. Farris. Taming Text: How to Find, Organize, and Manipulate It. New Jersey: Manning, 2013. Print.\\n\\nFuzzy Logic\\n\\nLogical reasoning that allows for degrees of truth for a statement.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Utilize when categories are not clearly deﬁned. Concepts such as \"warm\", \"cold\", and \"hot\" can mean different things at different temperatures and domains.\\n\\nZadeh L.A., \"Fuzzy Sets.” Information and Control. California: University of California, Berkeley, 1965. Print.\\n\\nGaussian Filtering\\n\\nActs to remove noise or blur data.\\n\\nCan be used to remove speckle noise from images.\\n\\nGeneralized Linear Models\\n\\nExpands ordinary linear regression to allow for error distribution that is not normal.\\n\\nUse if the observed error in your system does not follow the normal distribution.\\n\\nGenetic Algorithms\\n\\nEvolves candidate models over generations by evolutionary inspired operators of mutation and crossover of parameters.\\n\\nIncreasing the generation size adds diversity in considering parameter combinations, but requires more objective function evaluation. Calculating individuals within a generation is strongly parallelizable. Representation of candidate solutions can impact performance.\\n\\nGrid Search\\n\\nSystematic search across discrete parameter values for parameter exploration problems.\\n\\nA grid across the parameters is used to visualize the parameter landscape and assess whether multiple minima are present.\\n\\nParker, James R. Algorithms for Image Processing and Computer Vision. New Jersey: John Wiley & Sons, 2010. Print.\\n\\nMacCullagh, P., and John A. Nelder. Generalized Linear Models. Florida: CRC Press, 1989. Print.\\n\\nDe Jong, Kenneth A. Evolutionary Computation - A Uniﬁed Approach. Massachusetts: MIT Press, 2002. Print.\\n\\nKolda, Tamara G., Robert M. Lewis, and Virginia Torczon. “Optimization by Direct Search: New Perspectives on Some Classical and Modern Methods.” SIAM Review. Society for Industrial and Applied Mathematics, 2003. Print.\\n\\nBishop, Christopher M. Pattern Recognition and Machine Learning. New York: Springer, 2006. Print.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='One of the most powerful properties of Hidden Markov Models is their ability to exhibit some degree of invariance to local warping (compression and stretching) of the time axis. However, a signiﬁcant weakness of the Hidden Markov Model is the way in which it represents the distribution of times for which the system remains in a given state.\\n\\nProvides views of clusters at multiple resolutions of closeness. Algorithms begin to slow for larger datasets due to most implementations exhibiting O(N3) or O(N2) complexity.\\n\\nRui Xu, and Don Wunsch. Clustering. New Jersey: Wiley- IEEE Press, 2008. Print.\\n\\nWhen applying clustering techniques, make sure to understand the shape of your data. Clustering techniques will return poor results if your data is not circular or ellipsoidal shaped.\\n\\nRui Xu, and Don Wunsch. Clustering. New Jersey: Wiley- IEEE Press, 2008. Print.\\n\\nCompiled by: Booz Allen Hamilton\\n\\nHidden Markov Models\\n\\nModels sequential data by determining the discrete latent variables, but the observables may be continuous or discrete.\\n\\nHierarchical Clustering\\n\\nK\\n\\n\\n\\nmeans and X\\n\\n\\n\\nmeans Clustering\\n\\nConnectivity based clustering approach that sequentially builds bigger (agglomerative) or smaller (divisive) clusters in the data.\\n\\nCentroid based clustering algorithms, where with K means the number of clusters is set and X means the number of clusters is unknown.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nTechnique\\n\\nDescription\\n\\nTips From the Pros\\n\\nReferences we love to read\\n\\nLinear, Non\\n\\n\\n\\nlinear, and Integer Programming\\n\\nSet of techniques for minimizing or maximizing a function over a constrained set of input parameters.\\n\\nStart with linear programs because algorithms for integer and non-linear variables can take much longer to run.\\n\\nWinston, Wayne L. Operations Research: Applications and Algorithms. Connecticut: Cengage Learning, 2003. Print.\\n\\nMarkov Chain Monte Carlo (MCMC)', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='A method of sampling typically used in Bayesian models to estimate the joint distribution of parameters given the data.\\n\\nProblems that are intractable using analytic approaches can become tractable using MCMC, when even considering high-dimensional problems. The tractability is a result of using statistics on the underlying distributions of interest, namely, sampling with Monte Carlo and considering the stochastic sequential process of Markov Chains.\\n\\nAndrieu, Christophe, Nando de Freitas, Amaud Doucet, and Michael I. Jordan. “An Introduction to MCMC for Machine Learning.” Machine Learning. Kluwer Academic Publishers, 2003. Print.\\n\\nMonte Carlo Methods\\n\\nSet of computational techniques to generate random numbers.\\n\\nParticularly useful for numerical integration, solutions of differential equations, computing Bayesian posteriors, and high dimensional multivariate sampling.\\n\\nFishman, George S. Monte Carlo: Concepts, Algorithms, and Applications. New York: Springer, 2003. Print.\\n\\nNaïve Bayes\\n\\nPredicts classes following Bayes Theorem that states the probability of an outcome given a set of features is based on the probability of features given an outcome.\\n\\nAssumes that all variables are independent, so it can have issues learning in the context of highly interdependent variables. The model can be learned on a single pass of data using simple counts and therefore is useful in determining whether exploitable patterns exist in large datasets with minimal development time.\\n\\nIngersoll, Grant S., Thomas S. Morton, and Andrew L. Farris. Taming Text: How to Find, Organize, and Manipulate It. New Jersey: Manning, 2013. Print.\\n\\nNeural Networks\\n\\nLearns salient features in data by adjusting weights between nodes through a learning rule.\\n\\nTraining a neural network takes substantially longer than evaluating new data with an already trained network. Sparser network connectivity can help to segment the input space and improve performance on classiﬁcation tasks.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Haykin, Simon O. Neural Networks and Learning Machines. New Jersey: Prentice Hall, 2008. Print.\\n\\nOutlier Removal\\n\\nMethod for identifying and removing noise or artifacts from data.\\n\\nBe cautious when removing outliers. Sometimes the most interesting behavior of a system is at times when there are aberrant data points.\\n\\nMaimon, Oded, and Lior Rockach. Data Mining and Knowledge Discovery Handbook: A Complete Guide for Practitioners and Researchers. The Netherlands: Kluwer Academic Publishers, 2005. Print.\\n\\nPrincipal Components Analysis\\n\\nEnables dimensionality reduction by identifying highly correlated dimensions.\\n\\nMany large datasets contain correlations between dimensions; therefore part of the dataset is redundant. When analyzing the resulting principal components, rank order them by variance as this is the highest information view of your data. Use skree plots to infer the optimal number of components.\\n\\nWallisch, Pascal, Michael E. Lusignan, Marc D. Benayoun, Tanya I. Baker, Adam Seth Dickey, and Nicholas G. Hatsopoulos. Matlab for Neuroscientists. New Jersey: Prentice Hall, 2009. Print.\\n\\nRandom Search\\n\\nRandomly adjust parameters to ﬁnd a better solution than currently found.\\n\\nUse as a benchmark for how well a search algorithm is performing. Be careful to use a good random number generator and new seed.\\n\\nRegression with Shrinkage (Lasso)\\n\\nA method of variable selection and prediction combined into a possibly biased linear model.\\n\\nThere are different methods to select the lambda parameter. A typical choice is cross validation with MSE as the metric.\\n\\nSensitivity Analysis\\n\\nInvolves testing individual parameters in an analytic or model and observing the magnitude of the effect.\\n\\nInsensitive model parameters during an optimization are candidates for being set to constants. This reduces the dimensionality of optimization problems and provides an opportunity for speed up.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content=\"Bergstra J. and Bengio Y. Random Search for Hyper- Parameter Optimization, Journal of Machine Learning Research 13, 2012.\\n\\nTibshirani, Robert. “Regression Shrinkage and Selection via the Lasso.” Journal of the Royal Statistical Society. Series B (Methodological). Toronto: Royal Statistical Society, 1996. Print.\\n\\nSaltelli, A., Marco Ratto, Terry Andres, Francesca Campolongo, Jessica Cariboni, Debora Gatelli, Michaela Saisana, and Stefano Tarantola. Global Sensitivity Analysis: the Primer. New Jersey: John Wiley & Sons, 2008. Print.\\n\\nCompiled by: Booz Allen Hamilton\\n\\nTake off the Training Wheels\\n\\n81\\n\\nTechnique\\n\\nDescription\\n\\nTips From the Pros\\n\\nReferences we love to read\\n\\nSimulated Annealing\\n\\nStepwise Regression\\n\\nStochastic Gradient Descent\\n\\nNamed after a controlled cooling process in metallurgy, and by analogy using a changing temperature or annealing schedule to vary algorithmic convergence.\\n\\nA method of variable selection and prediction. Akaike's information criterion AIC is used as the metric for selection. The resulting predictive model is based upon ordinary least squares, or a general linear model with parameter estimation via maximum likelihood.\\n\\nGeneral-purpose optimization for learning of neural networks, support vector machines, and logistic regression models.\\n\\nThe standard annealing function allows for initial wide exploration of the parameter space followed by a narrower search. Depending on the search priority the annealing function can be modiﬁed to allow for longer explorative search at a high temperature.\\n\\nBertsimas, Dimitris, and John Tsitsiklis. “Simulated Annealing.” Statistical Science. 1993. Print.\\n\\nCaution must be used when considering Stepwise Regression, as over ﬁtting often occurs. To mitigate over ﬁtting try to limit the number of free variables used.\\n\\nHocking, R.R. “The Analysis and Selection of Variables in Linear Regression.” Biometrics. 1976. Print.\", metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Applied in cases where the objective function is not completely differentiable when using sub-gradients.\\n\\nSupport Vector Machines\\n\\nProjection of feature vectors using a kernel function into a space where classes are more separable.\\n\\nTry multiple kernels and use k-fold cross validation to validate the choice of the best one.\\n\\nTerm Frequency Inverse Document Frequency\\n\\nTopic Modeling (Latent Dirichlet Allocation)\\n\\nA statistic that measures the relative importance of a term from a corpus.\\n\\nTypically used in text mining. Assuming a corpus of news articles, a term that is very frequent such as “the” will likely appear many times in many documents, having a low value. A term that is infrequent such as a person’s last name that appears in a single article will have a higher TD IDF score.\\n\\nIngersoll, Grant S., Thomas S. Morton, and Andrew L. Farris. Taming Text: How to Find, Organize, and Manipulate It. New Jersey: Manning, 2013. Print.\\n\\nIdentiﬁes latent topics in text by examining word co-occurrence.\\n\\nEmploy part-of-speech tagging to eliminate words other than nouns and verbs. Use raw term counts instead of TF/IDF weighted terms.\\n\\nTree Based Methods\\n\\nModels structured as graph trees where branches indicate decisions.\\n\\nCan be used to systematize a process or act as a classiﬁer.\\n\\nT\\n\\n\\n\\nTest\\n\\nHypothesis test used to test for differences between two groups.\\n\\nMake sure you meet the tests assumptions and watch out for Family Wise error when running multiple tests.\\n\\nWrapper Methods\\n\\nFeature set reduction method that utilizes performance of a set of features on a model, as a measure of the feature set’s performance. Can help identify combinations of features in models that achieve high performance.\\n\\nUtilize k-fold cross validation to control over ﬁtting.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\nWitten, Ian H., Eibe Frank, and Mark A. Hall. Data Mining: Practical Machine Learning Tools and Techniques. Massachusetts: Morgan Kaufmann, 2011. Print.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Hsu, Chih-Wei, Chih-Chung Chang, and Chih-Jen Lin. “A Practical Guide to Support Vector Classiﬁcation.” National Taiwan University Press, 2003. Print.\\n\\nBlei, David M., Andrew Y. Ng, and Michael I. Jordan. “Latent Dirichlet Allocation.” Journal of Machine Learning Research. 2003. Print.\\n\\nJames, G., D. Witten, T. Hastie, and R. Tibshirani. “Tree Based Methods.” An Introduction to Statistical Learning. New York: Springer, 2013. Print.\\n\\nBhattacharyya, Gouri K., and Richard A. Johnson. Statistical Concepts and Models. Wiley, 1977. Print.\\n\\nJohn, George H., Ron Kohavi, and Karl Pﬂeger. “Irrelevant Features and the Subset Selection Problem.” Proceedings of ICML-94, 11th International Converence on Machine Learning. New Brunswick, New Jersey. 1994. Conference Presentation.\\n\\nCompiled by: Booz Allen Hamilton\\n\\nLI F E in T H E T R E N C H E S\\n\\nNAVIGATING NECK DEEP IN DATA\\n\\nOur Data Science experts have learned and developed new solutions over the years from properly framing or reframing analytic questions. In this section, we list a few important topics to Data Science coupled with firsthand experience from our experts.\\n\\nC L O S I N G T I M E\\n\\n››\\n\\nP A R T I N G T H O U G H T S\\n\\nData Science capabilities are creating data analytics that are touching every aspect of our lives on a daily basis. From visiting the doctor, to driving our cars, to shopping for services Data Science is quietly changing the way we interactive with and explore our world. We hope we have helped you truly understand the potential of your data and how to become extraordinary thinkers by asking the right questions. We hope we have helped continue to drive forward the science and art of Data Science. Most importantly, we hope you are leaving with a newfound passion and excitement for Data Science.', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Thank you for taking this journey with us. Please join our conversation and let your voice be heard. Email us your ideas and perspectives at data_science@bah.com or submit them via a pull request on the Github repository.\\n\\nTell us and the world what you know. Join us. Become an author of this story.\\n\\nClosing Time\\n\\n119\\n\\n››\\n\\nR E F E R E N C E S\\n\\n1. Commonly attributed to: Nye, Bill. Reddit Ask Me Anything (AMA).\\n\\nJuly 2012. Web. Accessed 15 October 2013. SSRN: <http://www.reddit. com/r/IAmA/comments/x9pq0/iam_bill_nye_the_science_guy_ama>\\n\\n2. Fayyad, Usama, Gregory Piatetsky-Shapiro, and Padhraic Smyth. “From Data Mining to Knowledge Discovery in Databases.” AI Magazine 17.3 (1996): 37-54. Print.\\n\\n3.\\n\\n“Mining Data for Nuggets of Knowledge.” Knowledge@Wharton,1999. Web. Accessed 16 October 2013. SSRN: <http://knowledge.wharton. upenn.edu/article/mining-data-for-nuggets-of-knowledge>\\n\\n4. Cleveland, William S. “Data Science: An Action Plan for Expanding the Technical Areas of the Field of Statistics.” International Statistical Review 69.1 (2001): 21-26. Print.\\n\\n5. Davenport, Thomas H., and D.J. Patil. “Data Scientist: The Sexiest Job of the 21st Century.” Harvard Business Review 90.10 (October 2012): 70–76. Print.\\n\\n6. Smith, David. “Statistics vs Data Science vs BI.” Revolutions, 15 May 2013. Web. Accessed 15 October 2013. SSRN:<http://blog. revolutionanalytics.com/2013/05/statistics-vs-data-science-vs-bi.html>\\n\\n7. Brynjolfsson, Erik, Lorin M. Hitt, and Heekyung H. Kim. “Strength in Numbers: How Does Data-Driven Decision Making Affect Firm Performance?” Social Science Electronic Publishing, 22 April 2011. Web. Accessed 15 October 2013. SSRN: <http://ssrn.com/abstract=1819486 or http://dx.doi.org/10.2139/ssrn.1819486>\\n\\n8.\\n\\n“The Stages of an Analytic Enterprise.” Nucleus Research. February 2012. Whitepaper.\\n\\n9. Barua, Anitesh, Deepa Mani, and Rajiv Mukherjee. “Measuring the', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Business Impacts of Effective Data.” University of Texas. Web. Accessed 15 October 2013. SSRNL: <http://www.sybase.com/files/White_Papers/ EffectiveDataStudyPt1-MeasuringtheBusinessImpactsofEffectiveDa ta-WP.pdf>\\n\\n10. Zikopoulos, Paul, Dirk deRoos, Kirshnan Parasuraman, Thomas Deutsch, David Corrigan and James Giles. Harness the Power of Big Data: The IBM Big Data Platform. New York: McGraw Hill, 2013. Print. 281pp.\\n\\nT H E F I ELD G U I D E to D A T A S C I E N C E\\n\\n11. Booz Allen Hamilton. Cloud Analytics Playbook. 2013. Web. Accessed 15 October 2013. SSRN: <http://www.boozallen.com/media/file/Cloud- playbook-digital.pdf>\\n\\n12. Conway, Drew. “The Data Science Venn Diagram.” March 2013.\\n\\nWeb. Accessed 15 October 2013. SSRN: <http://drewconway.com/ zia/2013/3/26/the-data-science-venn-diagram>\\n\\n13. Booz Allen Hamilton. Tips for Building a Data Science Capability. 2015.\\n\\nWeb Accessed 2 September 2015. SSRN: < https://www.boozallen.com/ content/dam/boozallen/documents/2015/07/DS-Capability-Handbook.pdf>\\n\\n14. Mnih et al. 2015. Human-level control through deep reinforcement\\n\\nlearning. Nature. 518: 529\\n\\n\\n\\n533.\\n\\n15. Torán, Jacobo. “On the Hardness of Graph Isomorphism.” SIAM Journal\\n\\non Computing. 33.5 (2004): 1093-1108. Print.\\n\\n16. Guyon, Isabelle and Andre Elisseeff. “An Introduction to Variable and Feature Selection.” Journal of Machine Learning Research 3 (March 2003):1157-1182. Print.\\n\\n17. Golub T., D. Slonim, P. Tamayo, C. Huard, M. Gaasenbeek, J. Mesirov, H. Coller, M. Loh, J. Downing, M. Caligiuri, C. Bloomfield, and E. Lander. “Molecular Classification of Cancer: Class Discovery and Class Prediction by Gene Expression Monitoring.” Science. 286.5439 (1999): 531-537. Print.\\n\\n18. Haykin, Simon O. Neural Networks and Learning Machines. New Jersey:\\n\\nPrentice Hall, 2008. Print.\\n\\n19. De Jong, Kenneth A. Evolutionary Computation - A Unified Approach.\\n\\nMassachusetts: MIT Press, 2002. Print.\\n\\n20. Yacci, Paul, Anne Haake, and Roger Gaborski. “Feature Selection of', metadata={'source': '/tmp/tmpp_bmeelb'}),\n",
              " Document(page_content='Microarray Data Using Genetic Algorithms and Artificial Neural Networks.” ANNIE 2009. St Louis, MO. 2-4 November 2009. Conference Presentation.\\n\\nClosing Time\\n\\n121\\n\\n››\\n\\nAbout B O O Z A LLEN H A M I L T O N\\n\\nBooz Allen Hamilton has been at the forefront of strategy and technology for more than 100 years. Today, the firm provides management and technology consulting and engineering services to leading Fortune 500 corporations, governments, and not-for-profits across the globe. Booz Allen partners with public and private sector clients to solve their most difficult challenges through a combination of consulting, analytics, mission operations, technology, systems delivery, cybersecurity, engineering, and innovation expertise.\\n\\nWith international headquarters in McLean, Virginia, the firm employs more than 22,500 people globally, and had revenue of $5.27 billion for the 12 months ended March 31, 2015. To learn more, visit www.boozallen.com. (NYSE: BAH)\\n\\nClosing Time\\n\\n123', metadata={'source': '/tmp/tmpp_bmeelb'})]"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "texts"
      ],
      "id": "BqDMaUjMVxfI"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F72QQmELV3j_",
        "outputId": "687d1288-e4c0-4969-d30b-05c5c24a6f3a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Document(page_content='Everyone you will ever meet knows something you don’t.\\n\\n[1]\\n\\n››\\n\\nT H E S TO RY\\n\\nof T H E F I E L D\\n\\nG U I D E\\n\\nSeveral years ago we created The Field Guide to Data Science because we wanted to help organizations of all types and sizes. There were countless industry and academic publications describing what Data Science is and why we should care, but very little information was available to explain how to make use of data as a resource. We find that situation to be just as true today as we did two years ago, when we created the first edition of the field guide.\\n\\nAt Booz Allen Hamilton, we built an industry-leading team of Data Scientists. Over the course of hundreds of analytic challenges for countless clients, we’ve unraveled the DNA of Data Science. Many people have put forth their thoughts on single aspects of Data Science. We believe we can offer a broad perspective on the conceptual models, tradecraft, processes and culture of Data Science – the what, the why, the who and the how. Companies with strong Data Science teams often focus on a single class of problems – graph algorithms for social network analysis, and recommender models for online shopping are two notable examples. Booz Allen is different. In our role as consultants, we support a diverse set of government and commercial clients across a variety of domains. This allows us to uniquely understand the DNA of Data Science.\\n\\nOur goal in creating The Field Guide to Data Science was to capture what we have learned and to share it broadly. The field of Data Science has continued to advance since we first released the field guide. As a result, we decided to release this second edition, incorporating a few new and important concepts. We also added technical depth and richness that we believe practitioners will find useful.\\n\\nWe want this effort to continue driving forward the science and art of Data Science.', metadata={'source': '/tmp/tmpp_bmeelb'})"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "texts[0]"
      ],
      "id": "F72QQmELV3j_"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "838b2843"
      },
      "source": [
        "### Create embeddings of your documents to get ready for semantic search"
      ],
      "id": "838b2843"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gMcCYHLRxgU8",
        "outputId": "9a29ab26-d7df-4f85-c780-0dcb0d357b09"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.2/177.2 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m23.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "argilla 1.7.0 requires pandas<2.0.0,>=1.0.0, but you have pandas 2.0.1 which is incompatible.\n",
            "google-colab 1.0.0 requires pandas==1.5.3, but you have pandas 2.0.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install -qU pinecone-client pandas"
      ],
      "id": "gMcCYHLRxgU8"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g1pMib9gWAOZ",
        "outputId": "80368276-e75f-4b9d-c85b-5cfb12112967"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Pinecone-client in /usr/local/lib/python3.10/dist-packages (2.2.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (2.27.1)\n",
            "Requirement already satisfied: pyyaml>=5.4 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (6.0)\n",
            "Requirement already satisfied: loguru>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (0.7.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (4.5.0)\n",
            "Requirement already satisfied: dnspython>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (2.3.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (2.8.2)\n",
            "Requirement already satisfied: urllib3>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (1.26.15)\n",
            "Requirement already satisfied: tqdm>=4.64.1 in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (4.65.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from Pinecone-client) (1.22.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.5.3->Pinecone-client) (1.16.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->Pinecone-client) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->Pinecone-client) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->Pinecone-client) (3.4)\n"
          ]
        }
      ],
      "source": [
        "!pip install Pinecone-client"
      ],
      "id": "g1pMib9gWAOZ"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o5q6A9tyYwAI"
      },
      "outputs": [],
      "source": [
        "!pip install -qU pinecone-client pandas"
      ],
      "id": "o5q6A9tyYwAI"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U7tRdmryWdX_",
        "outputId": "9f9fe32a-aaee-4c05-9db8-5b7ae342a4ed"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.27.7-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai) (2.27.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai) (4.65.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai) (3.8.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai) (3.4)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai) (1.3.1)\n",
            "Installing collected packages: openai\n",
            "Successfully installed openai-0.27.7\n"
          ]
        }
      ],
      "source": [
        "!pip install openai"
      ],
      "id": "U7tRdmryWdX_"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "373e695a",
        "outputId": "c0dd272d-8c64-4ca1-a75b-719155aac7b4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm\n"
          ]
        }
      ],
      "source": [
        "from langchain.vectorstores import Chroma, Pinecone\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "import pinecone"
      ],
      "id": "373e695a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0e093ef3"
      },
      "outputs": [],
      "source": [
        "\n",
        "# Check to see if there is an environment variable with you API keys, if not, use what you put below\n",
        "OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY', 'YourAPIKey')\n",
        "\n",
        "PINECONE_API_KEY = os.environ.get('PINECONE_API_KEY', 'YourAPIKey')\n",
        "PINECONE_API_ENV = os.environ.get('PINECONE_API_ENV', 'us-east1-gcp') # You may need to switch with your env"
      ],
      "id": "0e093ef3"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "4e0d1c6a"
      },
      "outputs": [],
      "source": [
        "embeddings = OpenAIEmbeddings(openai_api_key=OPENAI_API_KEY)"
      ],
      "id": "4e0d1c6a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0deb2f6a"
      },
      "outputs": [],
      "source": [
        "# initialize pinecone\n",
        "pinecone.init(\n",
        "    api_key=PINECONE_API_KEY, #find at app.pinecone.io\n",
        "    environment=PINECONE_API_ENV  # next to api key in console)\n",
        ")\n",
        "index_name = \"langchain\" # put in the name of your pinecone index here"
      ],
      "id": "0deb2f6a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "388988ce"
      },
      "outputs": [],
      "source": [
        "docsearch = Pinecone.from_texts([t.page_content for t in texts],embeddings, index_name=index_name)"
      ],
      "id": "388988ce"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "34929595"
      },
      "outputs": [],
      "source": [
        "query = \"What are examples of good data science teams?\"\n",
        "docs = docsearch.similarity_search(query)"
      ],
      "id": "34929595"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4e0f5b45",
        "outputId": "dd9c6755-fa9a-4fdd-9b2c-67e3796eee64"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Intelligence and cloud infrastructure development  \n",
            "work. We saw the need for a  \n",
            "new approach to distill value \n",
            "from our clients’ data. We \n",
            "approached the problem \n",
            "with a multidisciplinary \n",
            "team of computer scientists, \n",
            "mathematicians and domain \n",
            "experts. They immediately \n",
            "produced new insights and \n",
            "analysis paths, solidifying the \n",
            "validity of the approach. Since \n",
            "that time, our Data Science  \n",
            "team has grown to 250 staff \n",
            "supporting dozens of cl\n"
          ]
        }
      ],
      "source": [
        "# Here's an example of the first document that was returned\n",
        "print(docs[0].page_content[:450])"
      ],
      "id": "4e0f5b45"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3c35dcd9"
      },
      "source": [
        "### Query those docs to get your answer back"
      ],
      "id": "3c35dcd9"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f051337b"
      },
      "outputs": [],
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.chains.question_answering import load_qa_chain"
      ],
      "id": "f051337b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6b9b1c03"
      },
      "outputs": [],
      "source": [
        "llm = OpenAI(temperature=0, openai_api_key=OPENAI_API_KEY)\n",
        "chain = load_qa_chain(llm, chain_type=\"stuff\")"
      ],
      "id": "6b9b1c03"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f67ea7c2"
      },
      "outputs": [],
      "source": [
        "query = \"What is the collect stage of data maturity?\"\n",
        "docs = docsearch.similarity_search(query)"
      ],
      "id": "f67ea7c2"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3dfd2b7d",
        "outputId": "70ccf1e1-5c54-4ddb-f93a-ef103dfff624"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "' The collect stage of data maturity focuses on collecting internal or external datasets. Gathering sales records and corresponding weather data is an example of the collect stage.'"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chain.run(input_documents=docs, question=query)"
      ],
      "id": "3dfd2b7d"
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}