examples/efsync_scp_files.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/philschmid/efsync/blob/main/examples/efsync_scp_files.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "G2KJBvSLKmCy"
   },
   "source": [
    "# Example Notebook for efsync scp file upload\n",
    "\n",
    "this notebooks includes examples for the `CLI` and the `SDK`. \n",
    "\n",
    "This Notebook can be runned in google colab. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 601
    },
    "colab_type": "code",
    "id": "su7WWjnHKhVd",
    "outputId": "4d83a1ca-5009-41d1-f307-9d36a3cb9c9b",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting efsync\n",
      "  Downloading https://files.pythonhosted.org/packages/dc/6a/d2076c2778a56df0743e5361a9183423b1820d5230d19d43265456c8e972/efsync-0.4.0-py3-none-any.whl\n",
      "Collecting paramiko\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/19/124e9287b43e6ff3ebb9cdea3e5e8e88475a873c05ccdf8b7e20d2c4201e/paramiko-2.7.2-py2.py3-none-any.whl (206kB)\n",
      "\u001b[K     |████████████████████████████████| 215kB 5.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from efsync) (1.14.59)\n",
      "Collecting scp\n",
      "  Downloading https://files.pythonhosted.org/packages/4d/7a/3d76dc5ad8deea79642f50a572e1c057cb27e8b427f83781a2c05ce4e5b6/scp-0.13.2-py2.py3-none-any.whl\n",
      "Collecting pyaml\n",
      "  Downloading https://files.pythonhosted.org/packages/15/c4/1310a054d33abc318426a956e7d6df0df76a6ddfa9c66f6310274fb75d42/pyaml-20.4.0-py2.py3-none-any.whl\n",
      "Collecting pynacl>=1.0.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9d/57/2f5e6226a674b2bcb6db531e8b383079b678df5b10cdaa610d6cf20d77ba/PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl (961kB)\n",
      "\u001b[K     |████████████████████████████████| 962kB 8.1MB/s \n",
      "\u001b[?25hCollecting cryptography>=2.5\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c0/9c/647e559a6e8be493dc2a7a5d15d26cb501ca60ec299b356f23839a673a83/cryptography-3.1-cp35-abi3-manylinux2010_x86_64.whl (2.6MB)\n",
      "\u001b[K     |████████████████████████████████| 2.6MB 14.2MB/s \n",
      "\u001b[?25hCollecting bcrypt>=3.1.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/26/70/6d218afbe4c73538053c1016dd631e8f25fffc10cd01f5c272d7acf3c03d/bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl (63kB)\n",
      "\u001b[K     |████████████████████████████████| 71kB 6.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->efsync) (0.3.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->efsync) (0.10.0)\n",
      "Requirement already satisfied: botocore<1.18.0,>=1.17.59 in /usr/local/lib/python3.6/dist-packages (from boto3->efsync) (1.17.59)\n",
      "Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from pyaml->efsync) (3.13)\n",
      "Requirement already satisfied: cffi>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from pynacl>=1.0.1->paramiko->efsync) (1.14.2)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from pynacl>=1.0.1->paramiko->efsync) (1.15.0)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.59->boto3->efsync) (0.15.2)\n",
      "Requirement already satisfied: urllib3<1.26,>=1.20; python_version != \"3.4\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.59->boto3->efsync) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.59->boto3->efsync) (2.8.1)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.4.1->pynacl>=1.0.1->paramiko->efsync) (2.20)\n",
      "Installing collected packages: pynacl, cryptography, bcrypt, paramiko, scp, pyaml, efsync\n",
      "Successfully installed bcrypt-3.2.0 cryptography-3.1 efsync-0.4.0 paramiko-2.7.2 pyaml-20.4.0 pynacl-1.4.0 scp-0.13.2\n"
     ]
    }
   ],
   "source": [
    "!pip3 install efsync"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check installed version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "efsync==0.9.27\r\n"
     ]
    }
   ],
   "source": [
    "!pip freeze | grep efsync"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "usage: efsync <command> [<args>]\r\n",
      "\r\n",
      "optional arguments:\r\n",
      "  -h, --help            show this help message and exit\r\n",
      "  --version, -v         show program version\r\n",
      "  --requirements REQUIREMENTS, -r REQUIREMENTS\r\n",
      "                        path of your requirements.txt\r\n",
      "  --config_file CONFIG_FILE, -cf CONFIG_FILE\r\n",
      "                        path of your efsync.yaml\r\n",
      "  --python_version PYTHON_VERSION, -py PYTHON_VERSION\r\n",
      "                        Python version used to install dependencies\r\n",
      "  --efs_pip_dir EFS_PIP_DIR, -epd EFS_PIP_DIR\r\n",
      "                        directory where the pip packages will be installed on\r\n",
      "                        efs\r\n",
      "  --efs_filesystem_id EFS_FILESYSTEM_ID, -efi EFS_FILESYSTEM_ID\r\n",
      "                        File System ID from the EFS filesystem\r\n",
      "  --clean_efs CLEAN_EFS, -ce CLEAN_EFS\r\n",
      "                        Defines if the efs should be cleaned up before\r\n",
      "                        uploading\r\n",
      "  --file_dir FILE_DIR, -fd FILE_DIR\r\n",
      "                        directory where all other files will be placed\r\n",
      "  --aws_profile AWS_PROFILE, -ap AWS_PROFILE\r\n",
      "                        name of the used AWS profile\r\n",
      "  --aws_region AWS_REGION, -ar AWS_REGION\r\n",
      "                        aws region where the efs is running\r\n",
      "  --subnet_Id SUBNET_ID, -sbd SUBNET_ID\r\n",
      "                        subnet id of the efs\r\n",
      "  --ec2_key_name EC2_KEY_NAME, -ekn EC2_KEY_NAME\r\n",
      "                        temporary key name for the ec2 instance\r\n",
      "  --file_dir_on_ec2 FILE_DIR_ON_EC2, -fdoe FILE_DIR_ON_EC2\r\n",
      "                        name of the directory where your file from <file_dir>\r\n",
      "                        will be uploaded\r\n",
      "  --s3_bucket S3_BUCKET, -s3b S3_BUCKET\r\n",
      "                        s3 bucket name from where the files will be downloaded\r\n",
      "  --s3_keyprefix S3_KEYPREFIX, -s3k S3_KEYPREFIX\r\n",
      "                        s3 keyprefix of the directory in s3. Files will be\r\n",
      "                        downloaded\r\n"
     ]
    }
   ],
   "source": [
    "!efsync -h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from efsync import efsync"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example with `efsync.yaml`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting ./efsync.yaml\n"
     ]
    }
   ],
   "source": [
    "%%writefile ./efsync.yaml\n",
    "\n",
    "#standard configuration\n",
    "efs_filesystem_id: fs-2226b27a # aws efs filesystem id (moint point)\n",
    "subnet_Id: subnet-17f97a7d # subnet of which the efs is running in\n",
    "ec2_key_name: efsync-asd913fjgq3 # required key name for starting the ec2 instance\n",
    "clean_efs: all # Defines if the EFS should be cleaned up before. values: `'all'`,`'pip'`,`'file'` uploading\n",
    "# aws profile configuration\n",
    "aws_profile: schueler-vz # aws iam profile with required permission configured in .aws/credentials\n",
    "aws_region: eu-central-1 # the aws region where the efs is running\n",
    "\n",
    "# upload files with scp to efs\n",
    "file_dir: scp_data # extra local directory for file upload like ML models\n",
    "file_dir_on_ec2: ml # name of the directory where your file from <file_dir> will be uploaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-10-28 08:34:59,142 - efsync - starting....\n",
      "2020-10-28 08:34:59,143 - efsync - loading config\n",
      "2020-10-28 08:34:59,161 - efsync - creating security group\n",
      "2020-10-28 08:34:59,746 - efsync - loading default security group\n",
      "2020-10-28 08:34:59,931 - efsync - creating ssh key for scp in memory\n",
      "2020-10-28 08:35:00,073 - efsync - starting ec2 instance with security group sg-0544f2ee5fdbc55a8 and subnet_Id subnet-17f97a7d\n",
      "2020-10-28 08:37:43,332 - efsync - coping files from scp_data to /home/ec2-user/efs/ml \n",
      "2020-10-28 08:37:44,876 - efsync - stopping ec2 instance with instance id i-05632fd939f12b7d6\n",
      "2020-10-28 08:38:30,359 - efsync - deleting iam profile\n",
      "2020-10-28 08:38:31,965 - efsync - deleting ssh key\n",
      "2020-10-28 08:38:32,119 - efsync - deleting security group\n",
      "2020-10-28 08:38:32,351 - efsync - #################### finished after 3.5535 minutes ####################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from efsync import efsync\n",
    "\n",
    "efsync('./efsync.yaml')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example with pyton `dict`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "  'efs_filesystem_id': 'fs-2226b27a', # aws efs filesystem id (moint point)\n",
    "  'subnet_Id': 'subnet-17f97a7d', # subnet of which the efs is running in\n",
    "  'ec2_key_name':'efsync-asd913fjgq3',  # required key name for starting the ec2 instance\n",
    "  'clean_efs': 'all', # Defines if the EFS should be cleaned up before. values: `'all'`,`'pip'`,`'file'` uploading\n",
    "  'aws_profile': 'schueler-vz', # aws iam profile with required permission configured in .aws/credentials\n",
    "  'aws_region': 'eu-central-1', # the aws region where the efs is running\n",
    "  'file_dir': 'scp_data', # extra local directory for file upload like ML models\n",
    "  'file_dir_on_ec2': 'ml' # name of the directory where your file from <file_dir> will be uploaded\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-10-28 08:38:34,221 - efsync - starting....\n",
      "2020-10-28 08:38:34,222 - efsync - loading config\n",
      "2020-10-28 08:38:34,234 - efsync - creating security group\n",
      "2020-10-28 08:38:34,641 - efsync - loading default security group\n",
      "2020-10-28 08:38:34,818 - efsync - creating ssh key for scp in memory\n",
      "2020-10-28 08:38:35,023 - efsync - starting ec2 instance with security group sg-0da96cdf3911cd70f and subnet_Id subnet-17f97a7d\n",
      "2020-10-28 08:41:03,401 - efsync - coping files from scp_data to /home/ec2-user/efs/ml \n",
      "2020-10-28 08:41:04,857 - efsync - stopping ec2 instance with instance id i-0031e6c9c29c7a2d7\n",
      "2020-10-28 08:41:35,337 - efsync - deleting iam profile\n",
      "2020-10-28 08:41:36,495 - efsync - deleting ssh key\n",
      "2020-10-28 08:41:36,664 - efsync - deleting security group\n",
      "2020-10-28 08:41:36,895 - efsync - #################### finished after 3.0444999999999998 minutes ####################\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from efsync import efsync\n",
    "\n",
    "\n",
    "efsync(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example with cli and `efsync.yaml`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-10-28 08:41:37,372 - efsync - starting....\n",
      "2020-10-28 08:41:37,372 - efsync - loading config\n",
      "2020-10-28 08:41:37,384 - efsync - creating security group\n",
      "2020-10-28 08:41:37,760 - efsync - loading default security group\n",
      "2020-10-28 08:41:37,931 - efsync - creating ssh key for scp in memory\n",
      "2020-10-28 08:41:38,048 - efsync - starting ec2 instance with security group sg-0f8bfe18062a2bf9c and subnet_Id subnet-17f97a7d\n",
      "2020-10-28 08:44:05,837 - efsync - coping files from scp_data to /home/ec2-user/efs/ml \n",
      "2020-10-28 08:44:07,397 - efsync - stopping ec2 instance with instance id i-03a609557a033b18d\n",
      "2020-10-28 08:44:37,833 - efsync - deleting iam profile\n",
      "2020-10-28 08:44:38,904 - efsync - deleting ssh key\n",
      "2020-10-28 08:44:39,035 - efsync - deleting security group\n",
      "2020-10-28 08:44:39,273 - efsync - #################### finished after 3.0316666666666667 minutes ####################\n"
     ]
    }
   ],
   "source": [
    "!efsync -cf efsync.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example with cli and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-10-28 08:44:39,700 - efsync - starting....\n",
      "2020-10-28 08:44:39,700 - efsync - loading config\n",
      "2020-10-28 08:44:39,711 - efsync - creating security group\n",
      "2020-10-28 08:44:40,142 - efsync - loading default security group\n",
      "2020-10-28 08:44:40,323 - efsync - creating ssh key for scp in memory\n",
      "2020-10-28 08:44:40,440 - efsync - starting ec2 instance with security group sg-0623da6ebd41d7766 and subnet_Id subnet-17f97a7d\n",
      "2020-10-28 08:55:26,897 - efsync - coping files from scp_data to /home/ec2-user/efs/ml \n",
      "2020-10-28 08:55:28,370 - efsync - stopping ec2 instance with instance id i-067c3e0703c72e808\n",
      "2020-10-28 08:56:13,879 - efsync - deleting iam profile\n",
      "2020-10-28 08:56:14,888 - efsync - deleting ssh key\n",
      "2020-10-28 08:56:15,040 - efsync - deleting security group\n",
      "2020-10-28 08:56:15,283 - efsync - #################### finished after 11.593 minutes ####################\n"
     ]
    }
   ],
   "source": [
    "!efsync --efs_filesystem_id  fs-2226b27a \\\n",
    "        --subnet_Id subnet-17f97a7d \\\n",
    "        --ec2_key_name efsync-asd913fjgq3 \\\n",
    "        --clean_efs all \\\n",
    "        --aws_profile schueler-vz \\\n",
    "        --aws_region eu-central-1 \\\n",
    "        --file_dir scp_data \\\n",
    "        --file_dir_on_ec2 ml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qmaa_1XkK4e6"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPDRbc5orFAO1WZPC4Bxn2Y",
   "include_colab_link": true,
   "name": "efsync example sdk & cli.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}