notebooks/services/datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c52b5073",
   "metadata": {},
   "source": [
    "## Dataset API "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e97bf0ec",
   "metadata": {},
   "source": [
    "This API makes it easy to upload data to your cluster, such as for example a python program to run as a job. Also to download data from the cluster to your local environment."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "491b7181",
   "metadata": {},
   "source": [
    "## Scope"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6798df4a",
   "metadata": {},
   "source": [
    "This notebook showcases the following functionality:\n",
    "\n",
    "* Upload file\n",
    "* Download file\n",
    "* Check for file existence\n",
    "* Delete file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "118b6c52",
   "metadata": {},
   "outputs": [],
   "source": [
    "import hopsworks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c191b06d",
   "metadata": {},
   "source": [
    "## Connect to the cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6446746e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected. Call `.close()` to terminate connection gracefully.\n"
     ]
    }
   ],
   "source": [
    "# Connect to your cluster, to be used running inside Jupyter or jobs inside the cluster.\n",
    "connection = hopsworks.connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3a91b511",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment when connecting to the cluster from an external environment.\n",
    "# connection = hopsworks.connection(project='my_project', host='my_instance', port=443, api_key_value='apikey')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afd5c3cb",
   "metadata": {},
   "source": [
    "## Get the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eebf54bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the project object, if used inside your hopsworks cluster it gets the current project\n",
    "project = connection.get_project()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a47432b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment to get specific project\n",
    "# project = connection.get_project('my_project')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dc90f42",
   "metadata": {},
   "source": [
    "## Get the API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "913c5584",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_api = project.get_dataset_api()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b165b4ca",
   "metadata": {},
   "source": [
    "## Upload file to hopsworks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e2091bc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2022-04-12 12:57:52--  http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
      "Resolving yann.lecun.com (yann.lecun.com)... 188.114.96.7, 188.114.97.7, 2a06:98c1:3120::7, ...\n",
      "Connecting to yann.lecun.com (yann.lecun.com)|188.114.96.7|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 9912422 (9.5M) [application/x-gzip]\n",
      "Saving to: ‘train-images-idx3-ubyte.gz’\n",
      "\n",
      "train-images-idx3-u 100%[===================>]   9.45M  8.76MB/s    in 1.1s    \n",
      "\n",
      "2022-04-12 12:57:53 (8.76 MB/s) - ‘train-images-idx3-ubyte.gz’ saved [9912422/9912422]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Download example file to work with\n",
    "!wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "71182789",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8253cf9110854b3587279a07bc648a0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Uploading', max=9912422.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "File was uploaded on path: Resources/train-images-idx3-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Upload the file from the local environment to the cluster\n",
    "uploaded_file_path = dataset_api.upload(\"train-images-idx3-ubyte.gz\", \"Resources\", overwrite=True)\n",
    "print(\"File was uploaded on path: {}\".format(uploaded_file_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1b31b898",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check file existence\n",
    "dataset_api.exists(uploaded_file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40937345",
   "metadata": {},
   "source": [
    "## Download file from hopsworks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b0240edb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0cb17e8917043c58e0889d7c2e775f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=9912422.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'/srv/hops/staging/private_dirs/ca53d36092d3792645bc527eb455d405542efd398f2f62c130c1310416c1aa1e/train-images-idx3-ubyte.gz'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download the file from the cluster to the local environment\n",
    "dataset_api.download(uploaded_file_path, overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5bbe882f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the uploaded file or folder\n",
    "dataset_api.remove(uploaded_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "15fc523d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# File is now removed\n",
    "dataset_api.exists(uploaded_file_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}