notebooks/0_simple_bundle_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "import disdat.api as api\n",
    "from disdat.api import Bundle\n",
    "import tempfile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create example data context\n",
    "* Remove any existing examples for clean execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_context = 'example-context'\n",
    "api.context(data_context)\n",
    "api.rm(data_context, rm_all=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's make a bunch of versions of \"example_data\" \n",
    "* All example bundles will share the same \"human name\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bundle_name = \"example_data\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store scalar in a bundle with context manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with Bundle(data_context, name=bundle_name) as b:\n",
    "    b.add_data(True)\n",
    "    b.add_tags({'info':'storing a scalar'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store array in a bundle with constructor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = Bundle(data_context, name=bundle_name, data=[3,4,8,12,1000], tags={'info':'storing an array'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store a dictionary in a bundle manually\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = Bundle(data_context).open()\n",
    "b.name = bundle_name\n",
    "b.add_data({'jumping':[3.0,4.8], 'jack': [6,8,10]})\n",
    "b.add_tags({'info':'storing a dict'})\n",
    "b = b.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store existing local files in a bundle\n",
    "\n",
    "You can version existing files by storing the local file path (or s3 path see below).  \n",
    "\n",
    "Note: This copies the file into the local data context!\n",
    "\n",
    "* Return the file path as a string in b.data\n",
    "* Or place path in an array (or tuple), an array (or tuple) in a dictionary value, or in a dataframe column "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_fp = tempfile.NamedTemporaryFile()\n",
    "local_fp.write(b'an external local file')\n",
    "local_fp.flush()\n",
    "\n",
    "with Bundle(data_context, name=bundle_name) as b:\n",
    "    b.add_data(local_fp.name)\n",
    "    b.add_tags({'info':'added a local file'})\n",
    "\n",
    "local_fp.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Store existing S3 files in a bundle\n",
    "\n",
    "You can version existing S3 files by storing the S3 file path.  \n",
    "\n",
    "Note: requires valid AWS credentials\n",
    "\n",
    "* If you have a remote context, this copies the S3 file into the remote context.  \n",
    "* Otherwise it will copy the file into the local context. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "s3_file = 's3://landsat-pds/c1/L8/233/248/LC08_L1TP_233248_20170525_20170614_01_T1/LC08_L1TP_233248_20170525_20170614_01_T1_thumb_large.jpg'\n",
    "\n",
    "with Bundle(data_context, name=bundle_name) as b:\n",
    "    b.add_data(s3_file)\n",
    "    b.add_tags({'info':'copied in an s3 file'})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Avoiding file copies: Use managed paths to store a file directly in a bundle\n",
    "* Ask the bundle for a file target (places file in bundle directory)\n",
    "* Store the file in that file target\n",
    "* Return path or target in the bundle data field\n",
    "* Note: b.get_remote_file() will provide the same functionality when you have a remote context bound to the local context. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with Bundle(data_context, name=bundle_name) as b:\n",
    "    f1 = b.get_file(\"file_1.txt\")\n",
    "    f2 = b.get_file(\"file_2.txt\")\n",
    "    with open(f1, mode='w') as f:\n",
    "        f.write(\"This is our first file!\")\n",
    "    with open(f2,mode='w') as f:\n",
    "        f.write(\"This is our second file!\")\n",
    "    b.add_data([f1,f2])\n",
    "    b.add_tags({'info':'zero copy local file'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# List bundles with name \"example_data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "example_data\t2022-01-13 06:13:47.975175\n",
      "\tdata: ['/Users/kyocum/.disdat/context/example-context/objects/dfa2bc8c-571d-4572-8e79-7b89e8aa8d8a/file_1.txt'\n",
      " '/Users/kyocum/.disdat/context/example-context/objects/dfa2bc8c-571d-4572-8e79-7b89e8aa8d8a/file_2.txt']\n",
      "\tinfo: zero copy local file\n",
      "\n",
      "example_data\t2022-01-13 06:13:47.954023\n",
      "\tdata: s3://disdat-cdo-prd/context/example-context/objects/3dd0f0fb-1833-47d0-a086-65a275aaf263/LC08_L1TP_233248_20170525_20170614_01_T1_thumb_large.jpg\n",
      "\tinfo: copied in an s3 file\n",
      "\n",
      "example_data\t2022-01-13 06:13:46.840122\n",
      "\tdata: /Users/kyocum/.disdat/context/example-context/objects/48ed2208-285a-49f7-bc45-658b5bb4e26d/tmpd_rkmdlh\n",
      "\tinfo: added a local file\n",
      "\n",
      "example_data\t2022-01-13 06:13:46.819125\n",
      "\tdata: {'jumping': array([3. , 4.8]), 'jack': array([ 6,  8, 10])}\n",
      "\tinfo: storing a dict\n",
      "\n",
      "example_data\t2022-01-13 06:13:46.800036\n",
      "\tdata: [   3    4    8   12 1000]\n",
      "\tinfo: storing an array\n",
      "\n",
      "example_data\t2022-01-13 06:13:46.777935\n",
      "\tdata: True\n",
      "\tinfo: storing a scalar\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for b in api.search(data_context, bundle_name):    \n",
    "    print('{}\\t{}'.format(b.name, datetime.utcfromtimestamp(b.creation_date)))\n",
    "    print('\\tdata: {}'.format(b.data))\n",
    "    print('\\tinfo: {}'.format(b.tags['info']))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}