build_and_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# コンテナを利用して Lambda 関数を作成する\n",
    "このスクリプトを実行するための IAM の権限や、AWS configure とかは事前に要設定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, datetime, subprocess, json\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 様々な名前をユニークにするため、現在時刻のyyyymmddhhmmssを利用する\n",
    "now = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9))).strftime('%Y%m%d%H%M%S')\n",
    "image = 'lambda-container-yolov5'\n",
    "tag = ':latest'\n",
    "repository_name = f'{image}-{now}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ECR リポジトリを boto3 で作成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# ECR のリポジトリを作成する\n",
    "ecr = boto3.client('ecr')\n",
    "response = ecr.create_repository(\n",
    "    repositoryName=repository_name,\n",
    "    imageScanningConfiguration={'scanOnPush': True},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 必要な情報を抜き取っておく\n",
    "uri = response['repository']['repositoryUri']\n",
    "account_id = response['repository']['registryId']\n",
    "region = uri.split('.')[3]\n",
    "domain = uri.split('/')[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ECR にコンテナイメージをプッシュ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# build からプッシュまで\n",
    "\n",
    "# build 済なら実行不要\n",
    "# !docker build -t {image} .\n",
    "\n",
    "# タグ付与\n",
    "!docker tag {image}{tag} {uri}{tag}\n",
    "\n",
    "# ECR にログイン\n",
    "! aws ecr get-login-password | docker login --username AWS --password-stdin {domain}\n",
    "\n",
    "# push\n",
    "!docker push {uri}{tag}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 必要な情報を抜き取っておく\n",
    "res = ecr.describe_images(\n",
    "    repositoryName = repository_name\n",
    ")\n",
    "image_digest = res['imageDetails'][0]['imageDigest']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lambda を実行するロールを設定\n",
    "1. ロール作成\n",
    "2. ポリシー作成\n",
    "3. ポリシーをロールにアタッチ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam = boto3.client('iam')\n",
    "function_name = f'{image}-function-{now}'\n",
    "doc = {\n",
    "    'Version': '2012-10-17',\n",
    "    'Statement': [\n",
    "        {\n",
    "            'Action': 'sts:AssumeRole',\n",
    "            'Principal': {\n",
    "                'Service': 'lambda.amazonaws.com'\n",
    "                \n",
    "            },\n",
    "            'Effect': 'Allow',\n",
    "            'Sid': ''\n",
    "            \n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "# ロール作成\n",
    "role_name = f'{image}-role-{now}'\n",
    "res = iam.create_role(\n",
    "    Path = '/service-role/',\n",
    "    RoleName=role_name,\n",
    "    AssumeRolePolicyDocument=json.dumps(doc),\n",
    "    Description=f'exec role',\n",
    "    MaxSessionDuration=3600*12\n",
    ")\n",
    "role_arn = res['Role']['Arn']\n",
    "\n",
    "doc = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": \"logs:CreateLogGroup\",\n",
    "            \"Resource\": f\"arn:aws:logs:{account_id}:*\"\n",
    "        },\n",
    "        {\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Action\": [\n",
    "                \"logs:CreateLogStream\",\n",
    "                \"logs:PutLogEvents\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                f\"arn:aws:logs:{region}:{account_id}:log-group:/aws/lambda/{function_name}:*\"\n",
    "            ]\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "# ポリシー作成\n",
    "poicy_name = f'{image}-policy-{now}'\n",
    "res = iam.create_policy(\n",
    "    PolicyName=poicy_name,\n",
    "    PolicyDocument=json.dumps(doc),\n",
    ")\n",
    "policy_arn = res['Policy']['Arn']\n",
    "\n",
    "# 作成したポリシーをロールにアタッチ\n",
    "res = iam.attach_role_policy(\n",
    "    RoleName=role_name,\n",
    "    PolicyArn=policy_arn\n",
    ")\n",
    "\n",
    "# IAM の反映をしばし待つ\n",
    "sleep(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lambda 関数を作成する\n",
    "IAM の反映が間に合わない場合があるので、 sleep を入れているが、十分に時間が経っている場合は不要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# lambda function を Docker で作成したイメージから作成する\n",
    "lambda_client = boto3.client('lambda')\n",
    "res = lambda_client.create_function(\n",
    "    FunctionName=function_name,\n",
    "    Role=role_arn,\n",
    "    Code={\n",
    "        'ImageUri': f'{uri}@{image_digest}'\n",
    "    },\n",
    "    Description='input-> b64img, output -> b64img, yolov5 detect',\n",
    "    Timeout=60*15,\n",
    "    MemorySize=1024,\n",
    "    Publish=True,\n",
    "    PackageType='Image',\n",
    ")\n",
    "# 作成が完了するまで待つ\n",
    "while True:\n",
    "    res = lambda_client.get_function(FunctionName=function_name)\n",
    "    try:\n",
    "        if res['Configuration']['StateReasonCode']=='Creating':\n",
    "            print('.',end='')\n",
    "            sleep(1)\n",
    "    except:\n",
    "        if res['Configuration']['LastUpdateStatus']=='Successful':\n",
    "            print('!')\n",
    "            break\n",
    "        else:\n",
    "            print('?')\n",
    "            break\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 推論してみる"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import base64\n",
    "import requests\n",
    "import cv2\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "input_file = './yolov5/data/images/zidane.jpg'\n",
    "data = {}\n",
    "with open(input_file,'rb') as f:\n",
    "    data['img']= base64.b64encode(f.read()).decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "res = lambda_client.invoke(\n",
    "    FunctionName=function_name,\n",
    "    Payload=json.dumps(data)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_b64 = json.loads(res['Payload'].read())['img']\n",
    "img_bin = base64.b64decode(img_b64)\n",
    "img_array = np.frombuffer(img_bin,dtype=np.uint8)\n",
    "img = cv2.imdecode(img_array, cv2.IMREAD_COLOR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## お片付け"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_client.delete_function(\n",
    "    FunctionName=function_name\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam.detach_role_policy(\n",
    "    RoleName=role_name,\n",
    "    PolicyArn=policy_arn\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam.delete_policy(\n",
    "    PolicyArn=policy_arn\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam.delete_role(\n",
    "    RoleName=role_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ecr.batch_delete_image(\n",
    "    repositoryName=repository_name,\n",
    "    imageIds=[{\n",
    "        'imageDigest':image_digest\n",
    "    }]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ecr.delete_repository(\n",
    "    repositoryName=repository_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}