examples/mlrun_export_import.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exporting and Importing MLRun Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: ignore\n",
    "import nuclio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import zipfile\n",
    "from mlrun import DataItem\n",
    "\n",
    "def open_archive(context, \n",
    "                 target_dir: str,\n",
    "                 archive_url: DataItem = None):\n",
    "    \"\"\"Open a file/object archive into a target directory\n",
    "    \n",
    "    :param target_dir:   target directory\n",
    "    :param archive_url:  source archive path/url (MLRun DataItem object)\n",
    "    \n",
    "    :returns: content dir\n",
    "    \"\"\"\n",
    "        \n",
    "    # Define locations\n",
    "    archive_file = archive_url.local()\n",
    "    os.makedirs(target_dir, exist_ok=True)\n",
    "    context.logger.info('Verified directories')\n",
    "    \n",
    "    # Extract dataset from zip\n",
    "    context.logger.info('Extracting zip')\n",
    "    zip_ref = zipfile.ZipFile(archive_file, 'r')\n",
    "    zip_ref.extractall(target_dir)\n",
    "    zip_ref.close()\n",
    "    \n",
    "    context.logger.info(f'extracted archive to {target_dir}')\n",
    "    # use target_path= to specify and absolute target path (vs artifact_path)\n",
    "    context.log_artifact('content', target_path=target_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: end-code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export to a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create job function object from notebook code and add doc/metadata\n",
    "import mlrun\n",
    "fn = mlrun.code_to_function('file_utils', kind='job',\n",
    "                            handler='open_archive', image='mlrun/mlrun',\n",
    "                            description = \"this function opens a zip archive into a local/mounted folder\",\n",
    "                            categories = ['fileutils'],\n",
    "                            labels = {'author': 'me'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kind: job\n",
      "metadata:\n",
      "  name: file-utils\n",
      "  tag: ''\n",
      "  project: ''\n",
      "  labels:\n",
      "    author: me\n",
      "  categories:\n",
      "  - fileutils\n",
      "spec:\n",
      "  command: ''\n",
      "  args: []\n",
      "  image: mlrun/mlrun\n",
      "  volumes: []\n",
      "  volume_mounts: []\n",
      "  env: []\n",
      "  default_handler: open_archive\n",
      "  entry_points:\n",
      "    open_archive:\n",
      "      name: open_archive\n",
      "      doc: Open a file/object archive into a target directory\n",
      "      parameters:\n",
      "      - name: context\n",
      "      - name: target_dir\n",
      "        type: str\n",
      "        doc: target directory\n",
      "      - name: archive_url\n",
      "        doc: source archive path/url (MLRun DataItem object)\n",
      "      outputs:\n",
      "      - doc: content dir\n",
      "      lineno: 7\n",
      "  description: this function opens a zip archive into a local/mounted folder\n",
      "  build:\n",
      "    functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG9zCmltcG9ydCB6aXBmaWxlCmltcG9ydCBtbHJ1bgoKZGVmIG9wZW5fYXJjaGl2ZShjb250ZXh0LCAKICAgICAgICAgICAgICAgICB0YXJnZXRfZGlyOiBzdHIsCiAgICAgICAgICAgICAgICAgYXJjaGl2ZV91cmw6IG1scnVuLkRhdGFJdGVtID0gTm9uZSk6CiAgICAiIiJPcGVuIGEgZmlsZS9vYmplY3QgYXJjaGl2ZSBpbnRvIGEgdGFyZ2V0IGRpcmVjdG9yeQogICAgCiAgICA6cGFyYW0gdGFyZ2V0X2RpcjogICB0YXJnZXQgZGlyZWN0b3J5CiAgICA6cGFyYW0gYXJjaGl2ZV91cmw6ICBzb3VyY2UgYXJjaGl2ZSBwYXRoL3VybCAoTUxSdW4gRGF0YUl0ZW0gb2JqZWN0KQogICAgCiAgICA6cmV0dXJuczogY29udGVudCBkaXIKICAgICIiIgogICAgICAgIAogICAgYXJjaGl2ZV9maWxlID0gYXJjaGl2ZV91cmwubG9jYWwoKQogICAgb3MubWFrZWRpcnModGFyZ2V0X2RpciwgZXhpc3Rfb2s9VHJ1ZSkKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oJ1ZlcmlmaWVkIGRpcmVjdG9yaWVzJykKICAgIAogICAgY29udGV4dC5sb2dnZXIuaW5mbygnRXh0cmFjdGluZyB6aXAnKQogICAgemlwX3JlZiA9IHppcGZpbGUuWmlwRmlsZShhcmNoaXZlX2ZpbGUsICdyJykKICAgIHppcF9yZWYuZXh0cmFjdGFsbCh0YXJnZXRfZGlyKQogICAgemlwX3JlZi5jbG9zZSgpCiAgICAKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZidleHRyYWN0ZWQgYXJjaGl2ZSB0byB7dGFyZ2V0X2Rpcn0nKQogICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoJ2NvbnRlbnQnLCB0YXJnZXRfcGF0aD10YXJnZXRfZGlyKQoK\n",
      "    commands: []\n",
      "    code_origin: https://github.com/mlrun/mlrun.git#f1ae9dfaf2ba7cd6ba4188ff872cee1721d9c1c9:file_utils.ipynb\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(fn.to_yaml())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:50:59,815 function spec saved to path: function.yaml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.kubejob.KubejobRuntime at 0x7f91776d5c88>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save to a file (and can be pushed to a git)\n",
    "fn.export('function.yaml')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the function and run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlrun.mlconf.dbpath = mlrun.mlconf.dbpath or 'http://mlrun-api:8080'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "function: file-utils\n",
      "this function opens a zip archive into a local/mounted folder\n",
      "default handler: open_archive\n",
      "entry points:\n",
      "  open_archive: Open a file/object archive into a target directory\n",
      "    context  - \n",
      "    target_dir(str)  - target directory\n",
      "    archive_url  - source archive path/url (MLRun DataItem object)\n"
     ]
    }
   ],
   "source": [
    "# load from local file\n",
    "xfn = mlrun.import_function('./function.yaml')\n",
    "\n",
    "# load function from MLRun functions hub \n",
    "# xfn = mlrun.import_function('hub://open_archive')\n",
    "\n",
    "# get function doc\n",
    "xfn.doc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import path\n",
    "from mlrun.platforms import auto_mount\n",
    "# for auto choice between Iguazio platform and k8s PVC\n",
    "# should set the env var for PVC: MLRUN_PVC_MOUNT=<pvc-name>:<mount-path>, or use mount_pvc() \n",
    "xfn.apply(auto_mount())\n",
    "\n",
    "# create and run the task\n",
    "images_path = path.abspath('images')\n",
    "open_archive_task = mlrun.new_task('download',\n",
    "    params={'target_dir': images_path},\n",
    "    inputs={'archive_url': 'http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:51:03,218 artifact path is not defined or is local, artifacts will not be visible in the UI\n",
      "[mlrun] 2020-06-08 21:51:03,226 starting run download uid=1a4c43546ad7437ea8b9055e601c9fad  -> http://mlrun-api:8080\n",
      "[mlrun] 2020-06-08 21:51:03,259 starting local run: /tmp/tmpq7o4xvch.py # open_archive\n",
      "[mlrun] 2020-06-08 21:51:03,276 downloading http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip to local tmp\n",
      "[mlrun] 2020-06-08 21:51:04,365 Verified directories\n",
      "[mlrun] 2020-06-08 21:51:04,365 Extracting zip\n",
      "[mlrun] 2020-06-08 21:51:11,813 extracted archive to /User/mlrun/examples/images\n",
      "[mlrun] 2020-06-08 21:51:11,830 log artifact content at /User/mlrun/examples/images, size: None, db: Y\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"1a4c43546ad7437ea8b9055e601c9fad\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/1a4c43546ad7437ea8b9055e601c9fad/info\" target=\"_blank\" >...601c9fad</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 21:51:03</td>\n",
       "      <td>completed</td>\n",
       "      <td>download</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=jupyter-65887d7ffb-5jsn2</div></td>\n",
       "      <td><div title=\"http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip\">archive_url</div></td>\n",
       "      <td><div class=\"dictlist\">target_dir=/User/mlrun/examples/images</div></td>\n",
       "      <td></td>\n",
       "      <td><div title=\"/User/mlrun/examples/images\">content</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resulta1b320e8-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resulta1b320e8-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resulta1b320e8\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resulta1b320e8-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run 1a4c43546ad7437ea8b9055e601c9fad --project default , !mlrun logs 1a4c43546ad7437ea8b9055e601c9fad --project default\n",
      "[mlrun] 2020-06-08 21:51:11,877 run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "run = mlrun.run_local(open_archive_task, xfn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run as a cluster job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlrun import mlconf\n",
    "mlconf.dbpath = mlconf.dbpath or './'\n",
    "artifact_path = mlconf.artifact_path or path.abspath('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:51:11,900 starting run download uid=70ac5c42098b49a1965ff80eb3bed2e7  -> http://mlrun-api:8080\n",
      "[mlrun] 2020-06-08 21:51:11,983 Job is running in the background, pod: download-fbm52\n",
      "[mlrun] 2020-06-08 21:51:16,068 starting local run: main.py # open_archive\n",
      "[mlrun] 2020-06-08 21:51:16,083 downloading http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip to local tmp\n",
      "[mlrun] 2020-06-08 21:51:16,949 Verified directories\n",
      "[mlrun] 2020-06-08 21:51:16,950 Extracting zip\n",
      "[mlrun] 2020-06-08 21:51:24,350 extracted archive to /User/mlrun/examples/images\n",
      "[mlrun] 2020-06-08 21:51:24,364 log artifact content at /User/mlrun/examples/images, size: None, db: Y\n",
      "\n",
      "[mlrun] 2020-06-08 21:51:24,375 run executed, status=completed\n",
      "final state: succeeded\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"70ac5c42098b49a1965ff80eb3bed2e7\"><a href=\"https://mlrun-ui.default-tenant.app.yh55.iguazio-cd2.com/projects/default/jobs/70ac5c42098b49a1965ff80eb3bed2e7/info\" target=\"_blank\" >...b3bed2e7</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Jun 08 21:51:16</td>\n",
       "      <td>completed</td>\n",
       "      <td>download</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=download-fbm52</div></td>\n",
       "      <td><div title=\"http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip\">archive_url</div></td>\n",
       "      <td><div class=\"dictlist\">target_dir=/User/mlrun/examples/images</div></td>\n",
       "      <td></td>\n",
       "      <td><div title=\"/User/mlrun/examples/images\">content</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"resultc7e86725-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"resultc7e86725-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"resultc7e86725\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"resultc7e86725-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run 70ac5c42098b49a1965ff80eb3bed2e7 --project default , !mlrun logs 70ac5c42098b49a1965ff80eb3bed2e7 --project default\n",
      "[mlrun] 2020-06-08 21:51:27,212 run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "run = xfn.run(open_archive_task, artifact_path=artifact_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading and running functions as local python modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlrun import function_to_module, get_or_create_ctx\n",
    "mod = function_to_module(xfn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a context object and DataItem objects\n",
    "# you can also use existing context and data objects (e.g. from parant function)  \n",
    "context = get_or_create_ctx('myfunc')\n",
    "data = mlrun.run.get_dataitem('http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-06-08 21:51:27,245 downloading http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip to local tmp\n",
      "[mlrun] 2020-06-08 21:51:28,099 Verified directories\n",
      "[mlrun] 2020-06-08 21:51:28,100 Extracting zip\n",
      "[mlrun] 2020-06-08 21:51:35,622 extracted archive to /User/mlrun/examples/images\n",
      "[mlrun] 2020-06-08 21:51:35,637 log artifact content at /User/mlrun/examples/images, size: None, db: Y\n"
     ]
    }
   ],
   "source": [
    "mod.open_archive(context, target_dir=images_path, archive_url=data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kind: run\n",
      "metadata:\n",
      "  name: download\n",
      "  uid: 1a4c43546ad7437ea8b9055e601c9fad\n",
      "  iteration: 0\n",
      "  project: default\n",
      "  labels:\n",
      "    v3io_user: admin\n",
      "    kind: ''\n",
      "    owner: admin\n",
      "    host: jupyter-65887d7ffb-5jsn2\n",
      "  annotations: {}\n",
      "spec:\n",
      "  function: /file-utils\n",
      "  log_level: info\n",
      "  parameters:\n",
      "    target_dir: /User/mlrun/examples/images\n",
      "  outputs: []\n",
      "  output_path: ''\n",
      "  inputs:\n",
      "    archive_url: http://iguazio-sample-data.s3.amazonaws.com/catsndogs.zip\n",
      "  data_stores: []\n",
      "status:\n",
      "  state: completed\n",
      "  results: {}\n",
      "  start_time: '2020-06-08T21:51:03.261376+00:00'\n",
      "  last_update: '2020-06-08T21:51:11.831849+00:00'\n",
      "  artifacts:\n",
      "  - key: content\n",
      "    kind: ''\n",
      "    iter: 0\n",
      "    tree: 1a4c43546ad7437ea8b9055e601c9fad\n",
      "    target_path: /User/mlrun/examples/images\n",
      "    db_key: download_content\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(context.to_yaml())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}