python advanced topic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Python Advanced Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ Functional Programming, decorator\n",
    "+ Python Profiler;\n",
    "+ RE\n",
    "+ Multiprocessing Programming\n",
    "+ \\__buid\\__in\n",
    "+ Syntax Tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ Unrelated-to-language / Algorithm, Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ related-to-lanauge / (Python, Java) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ assert \n",
    "+ *arg, **kwargs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Functional Programming"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Oriented Functional Programming"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Oriented Object Programming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Student: \n",
    "    def __init__(): \n",
    "        # properties\n",
    "        pass\n",
    "    \n",
    "    def say_hello(): \n",
    "        pass\n",
    "    \n",
    "    def take_a_test():\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_a_studnet_info(student): pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Student.say_hello()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "say_hello(student)\n",
    "# binding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Oriented XX\n",
    "    + Focus\n",
    "    + 1. XX Var; 2. XX Arg 3. XX Return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_number_by_func(9, 3, lambda x, y: x + y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def less_than_zero(x): return x < 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_is_even_number_by_func(arg1, arg2, operation_f):\n",
    "    return operation_f(arg1, arg2) % 2 == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# need to check if the arg1 and arg2 's result is less than zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_is_even_number_by_func(arg1, arg2, operation_f):\n",
    "    return operation_f(arg1, arg2) < 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def add_two_number(a, b): return a + b\n",
    "\n",
    "check_number_by_func(9, 3, add_two_number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def some_func(x, y): return x! + y!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "lambda arg1, arg2, ... : # some operation with args\n",
    "+ if / else / while / for \n",
    "+ return \n",
    "+ assignment\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_number_by_func(9, 3, lambda x, y: some_func(x, y) + 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_primer(n):\n",
    "    for i in range(2, n):\n",
    "        if n % i == 0: return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 2.6226043701171875e-06\n",
      "used time: 0.00012493133544921875\n",
      "used time: 2.1457672119140625e-06\n",
      "used time: 3.314018249511719e-05\n",
      "used time: 2.1457672119140625e-06\n",
      "used time: 2.002716064453125e-05\n",
      "used time: 1.9073486328125e-06\n",
      "used time: 2.3126602172851562e-05\n",
      "used time: 8.106231689453125e-06\n",
      "used time: 3.528594970703125e-05\n",
      "used time: 2.1457672119140625e-06\n",
      "used time: 3.0040740966796875e-05\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_the_closet_points(points_pair):\n",
    "    min_distance, closet_pair = float('inf'), None\n",
    "    for p1, p2 in combinations(points_pair, 2):\n",
    "        # distance = sqrt((x1 - x2) ^ 2 + (y1 - y2) ^ 2)\n",
    "        delta = math.sqrt((p1[0]-p2[0])**2 + (p1[1] - p2[1]) ** 2) \n",
    "        if delta < min_distance:\n",
    "            min_distance = delta\n",
    "            closet_pair = (p1, p2)\n",
    "    return p1, p2, min_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5, 3), (3, 2), 1.0)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_the_closet_points([(0, 0), (1, 1), (3, 3), (5, 3), (3, 2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 4.9114227294921875e-05\n"
     ]
    }
   ],
   "source": [
    "s = time.time()\n",
    "is_primer(33123141235245345331)\n",
    "print('used time: {}'.format(time.time() - s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_running_time_old(func, *args, **kwargs):\n",
    "    s = time.time()\n",
    "    result = func(*args, **kwargs)\n",
    "    print('used time: {}'.format(time.time() - s))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_running_time(func): \n",
    "    @wraps(func)\n",
    "    def wrapper(*args, **kwargs):\n",
    "        s = time.time()\n",
    "        result = func(*args, **kwargs)\n",
    "        print('used time: {}'.format(time.time() - s))\n",
    "        return result\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_primer = get_running_time(is_primer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 1.0990970134735107\n"
     ]
    }
   ],
   "source": [
    "result = is_primer(9038318248193747234825891)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 1.1440272331237793\n"
     ]
    }
   ],
   "source": [
    "result_1 = get_running_time_old(is_primer, 9038318248193747234825891)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import wraps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "def send_mail(func_name): print('send mail to moniter')\n",
    "\n",
    "\n",
    "\n",
    "def notify(func):\n",
    "    @wraps(func)\n",
    "    def wrapper(*args, **kwargs):\n",
    "        \"\"\"the notify wrapper\"\"\"\n",
    "        send_mail(func.__name__)\n",
    "        result = func(*args, **kwargs)\n",
    "        return result\n",
    "    return wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# python decorator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_primer = notify(is_primer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 100000 -> 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext line_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mod(n, i): return n % i\n",
    "\n",
    "def is_primer(n):\n",
    "    \"\"\"check if is a primer\"\"\"\n",
    "    s = int(math.sqrt(n))\n",
    "    \n",
    "    if n > 2 and n % 2 == 0: return False\n",
    "    if n > 3 and n % 3 == 0: return False\n",
    "    if n > 5 and n % 5 == 0: return False\n",
    "\n",
    "    for i in range(6, n):\n",
    "        if mod(n, i) == 0: return False\n",
    "    return True\n",
    "\n",
    "assert is_primer(3)\n",
    "assert not is_primer(4)\n",
    "assert not is_primer(10)\n",
    "assert is_primer(13)\n",
    "assert is_primer(97)\n",
    "assert not is_primer(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_primers(max_n):\n",
    "    result = []\n",
    "    for i in range(2, max_n+1):\n",
    "        if is_primer(i): \n",
    "            result.append(i)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "|2 2|\n",
    "|3 4|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fast_det(A):\n",
    "    return A[0][0]*A[1][1] - A[0][1]*A[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-299-0dc433b3bb8b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100000000000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for _ in range(100000000000): pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** KeyboardInterrupt exception caught in code being profiled."
     ]
    }
   ],
   "source": [
    "%lprun -f is_primer get_primers(50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "11   1061109     379044.0      0.4     35.1      for i in range(2, s+1):\n",
    "12   1055976     663132.0      0.6     61.4          if mod(n, i) == 0: return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "11 1006118     359130.0      0.4     33.3      for i in range(2, s+1):\n",
    "12 1000985     638955.0      0.6     59.2          if mod(n, i) == 0: return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " 7     49999      24586.0      0.5      2.3      if n > 2 and n % 2 == 0: return False\n",
    " 8     25000      11297.0      0.5      1.1      if n > 3 and n % 3 == 0: return False\n",
    " 9     16668       7498.0      0.4      0.7      if n > 5 and n % 5 == 0: return False\n",
    "10                                           \n",
    "11    952795     345700.0      0.4     33.0      for i in range(6, s+1):\n",
    "12    947662     618787.0      0.7     59.1          if mod(n, i) == 0: return False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 151 ms, sys: 3.4 ms, total: 154 ms\n",
      "Wall time: 154 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "result = get_primers(50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "send mail to moniter\n",
      "used time: 6.198883056640625e-06\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_primer(9231823901823877988778)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function is_primer in module __main__:\n",
      "\n",
      "is_primer(n)\n",
      "    check if is a primer\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(is_primer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Functional Programming\n",
    "2. Decorator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# decrease complexity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_primer = get_running_time(is_primer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_the_closet_points  = get_running_time(get_the_closet_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 0.4699971675872803\n"
     ]
    }
   ],
   "source": [
    "result = get_the_closet_points([(random.randrange(100), random.randrange(100)) for _ in range(1000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((71, 8), (51, 35), 0.0)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 5.0067901611328125e-06\n"
     ]
    }
   ],
   "source": [
    "result = is_primer(290038310983192083)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "used time: 0.43616724014282227\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((37, 34), (26, 22), 0.0)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_running_time_old(get_the_closet_points, [(random.randrange(100), random.randrange(100)) for _ in range(1000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://github.com/Deep-Learning-for-NLP-Chinese'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = 'I am a good boy, and I am always drink milk, and I drink 21 kg milk each day'\n",
    "example_2 = 'I am a girl, I dont drink milk'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 527,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = '(.*boy.*milk.*\\s(\\d+).*)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 523,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_3 = 'I am a good boy, and I am always drink milk, and I drink milk each day'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 525,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I am a good boy, and I am always drink milk, and I drink 21 kg milk each day',\n",
       "  '21')]"
      ]
     },
     "execution_count": 525,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(pattern, example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(pattern, example_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I am a good boy, and I am always drink milk, and I drink 2 kg milk each day']"
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(pattern, example)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(pattern, example_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example.find('milk') > 0 and example.find('boy') > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_pattern = '.*href=\"(/Deep-Learning-for-NLP-Chinese/.*)\"\\s.*'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Deep-Learning-for-NLP-Chinese/assignment2-Project2']"
      ]
     },
     "execution_count": 371,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(url_pattern,\n",
    "    '<a href=\"/Deep-Learning-for-NLP-Chinese/assignment2-Project2\" itemprop=\"name codeRepository\">')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Deep-Learning-for-NLP-Chinese/assignment2-Project2',\n",
       " '/Deep-Learning-for-NLP-Chinese/tensorflow',\n",
       " '/Deep-Learning-for-NLP-Chinese/Slides',\n",
       " '/Deep-Learning-for-NLP-Chinese/Word2Vec',\n",
       " '/Deep-Learning-for-NLP-Chinese/References',\n",
       " '/Deep-Learning-for-NLP-Chinese/assignment_get_douban_comment',\n",
       " '/Deep-Learning-for-NLP-Chinese/Project-1',\n",
       " '/Deep-Learning-for-NLP-Chinese/Data_source',\n",
       " '/Deep-Learning-for-NLP-Chinese/Intro',\n",
       " '/Deep-Learning-for-NLP-Chinese/DeepNeuralNetworks',\n",
       " '/Deep-Learning-for-NLP-Chinese/Sequence2Sequence',\n",
       " '/Deep-Learning-for-NLP-Chinese/ReinforcementLearning',\n",
       " '/Deep-Learning-for-NLP-Chinese/ReadingComprehension',\n",
       " '/Deep-Learning-for-NLP-Chinese/AutoSummarization',\n",
       " '/Deep-Learning-for-NLP-Chinese/ChatBots',\n",
       " '/Deep-Learning-for-NLP-Chinese/TreeRNN',\n",
       " '/Deep-Learning-for-NLP-Chinese/SentenceEmbedding',\n",
       " '/Deep-Learning-for-NLP-Chinese/CNN',\n",
       " '/Deep-Learning-for-NLP-Chinese/RNN-LSTM-GRU']"
      ]
     },
     "execution_count": 378,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(url_pattern, r.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Syntax Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'你好，请问你要办理什么业务？'\n",
    "'先生您好，'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'你好，您需要办理开户业务吗？'"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'你好，您需要办理开户业务吗？'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = 寒暄，询问，业务相关"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \"\"\"\n",
    "sentence = 寒暄 报数 询问 业务相关 结尾\n",
    "报数 = 我是 数字 号 ,\n",
    "数字 = num num | num\n",
    "num = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9\n",
    "寒暄 = 称谓 打招呼 | 打招呼\n",
    "称谓 = 人称 ,\n",
    "人称 = 先生 | 女士 | 小朋友\n",
    "打招呼 = 你好 | 您好 \n",
    "询问 = 请问你要办理 | 您需要办理\n",
    "业务相关 = 具体业务 业务\n",
    "具体业务 = 开户 | 理财 | 挂失 | 信用卡\n",
    "结尾 = 吗？| 呢？\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Complier Principle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 510,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = {}\n",
    "\n",
    "for line in sentence.split('\\n'):\n",
    "    if not line: continue\n",
    "    key, rules = line.split('=')\n",
    "    key = key.strip()\n",
    "    rules = rules.split('|')\n",
    "    pattern[key] = rules\n",
    "    pattern[key] = [r.split() for r in rules]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence': [['寒暄', '报数', '询问', '业务相关', '结尾']],\n",
       " '报数': [['我是', '数字', '号', ',']],\n",
       " '数字': [['num', 'num'], ['num']],\n",
       " 'num': [['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']],\n",
       " '寒暄': [['称谓', '打招呼'], ['打招呼']],\n",
       " '称谓': [['先生'], ['女士'], ['小朋友']],\n",
       " '打招呼': [['你好'], ['您好']],\n",
       " '询问': [['请问你要办理'], ['您需要办理']],\n",
       " '业务相关': [['具体业务', '业务']],\n",
       " '具体业务': [['开户'], ['理财'], ['挂失']],\n",
       " '结尾': [['吗？'], ['呢？']]}"
      ]
     },
     "execution_count": 468,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate(target='sentence'):\n",
    "    if target not in pattern: return target\n",
    "    \n",
    "    rule = random.choice(pattern[target])\n",
    "    return ''.join(generate(r) for r in rule)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Driven"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 520,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'女士,您好我是7号,您需要办理开户业务吗？'"
      ]
     },
     "execution_count": 520,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate('sentence')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}