chunks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-11T00:34:41.434656Z",
     "start_time": "2019-04-11T00:34:41.420985Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "尬\n",
      "了\n",
      "看\n",
      " \n",
      "少\n",
      "”\n"
     ]
    }
   ],
   "source": [
    "def read_in_chunks(file_object, chunk_size=1024):\n",
    "    \"\"\"Lazy function (generator) to read a file piece by piece.\n",
    "    Default chunk size: 1k.\"\"\"\n",
    "    while True:\n",
    "        data = file_object.read(chunk_size)\n",
    "        if not data:\n",
    "            break\n",
    "        yield data\n",
    "\n",
    "\n",
    "f = open('f:/github/baiduAPI/泼辣小厨娘.txt','r',encoding='utf-8')\n",
    "for piece in read_in_chunks(f,chunk_size=100000):\n",
    "    print(piece[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-01T02:10:19.872177Z",
     "start_time": "2019-09-01T02:10:19.865197Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[72, 87, 21]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[72, 87, 21, 6, 86, 78, 45, 46, 61, 99]\n",
      "[33, 88, 88, 47, 73, 39, 56, 12, 84, 71]\n",
      "[15, 50, 11, 92, 89, 15, 14, 67, 67, 96]\n",
      "[84]\n"
     ]
    }
   ],
   "source": [
    "l = list(np.random.randint(3, 100, 31))\n",
    "l[:3]\n",
    "num = 10\n",
    "l_chunks = [l[i:i+num] for i in range(0, len(l), num)]\n",
    "for i in l_chunks:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-01T02:10:21.689166Z",
     "start_time": "2019-09-01T02:10:21.684180Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 72, 1: 87, 2: 21, 3: 6, 4: 86, 5: 78, 6: 45, 7: 46, 8: 61, 9: 99, 10: 33, 11: 88, 12: 88, 13: 47, 14: 73, 15: 39, 16: 56, 17: 12, 18: 84, 19: 71, 20: 15, 21: 50, 22: 11, 23: 92, 24: 89, 25: 15, 26: 14, 27: 67, 28: 67, 29: 96, 30: 84}\n"
     ]
    }
   ],
   "source": [
    "d = {}\n",
    "for id, v in enumerate(l):\n",
    "    d[id] = v\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-01T02:10:25.488157Z",
     "start_time": "2019-09-01T02:10:25.484108Z"
    }
   },
   "outputs": [],
   "source": [
    "l_keys = [list(d.keys())[i:i+num] for i in range(0, len(d), num)]\n",
    "l_values = [list(d.values())[i:i+num] for i in range(0, len(d), num)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-01T02:10:25.943174Z",
     "start_time": "2019-09-01T02:10:25.940149Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
      "[72, 87, 21, 6, 86, 78, 45, 46, 61, 99]\n"
     ]
    }
   ],
   "source": [
    "print(l_keys[0] )\n",
    "print(l_values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-01T02:12:16.353913Z",
     "start_time": "2019-09-01T02:12:16.348927Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{0: 72, 1: 87, 2: 21, 3: 6, 4: 86, 5: 78, 6: 45, 7: 46, 8: 61, 9: 99}, {10: 33, 11: 88, 12: 88, 13: 47, 14: 73, 15: 39, 16: 56, 17: 12, 18: 84, 19: 71}, {20: 15, 21: 50, 22: 11, 23: 92, 24: 89, 25: 15, 26: 14, 27: 67, 28: 67, 29: 96}, {30: 84}]\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "dict_l = [dict(zip(k, v)) for k, v in zip(l_keys,l_values)]\n",
    "print(dict_l)\n",
    "print(len(dict_l))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}