4.1-文本数据样本采集.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "371B074F2456405FA34261A9B10AA291",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original_seq:  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]\n",
      "x:  tensor([[ 5,  6,  7,  8,  9],\n",
      "        [15, 16, 17, 18, 19],\n",
      "        [ 0,  1,  2,  3,  4],\n",
      "        [20, 21, 22, 23, 24]])\n",
      "y:  tensor([[ 6,  7,  8,  9, 10],\n",
      "        [16, 17, 18, 19, 20],\n",
      "        [ 1,  2,  3,  4,  5],\n",
      "        [21, 22, 23, 24, 25]])\n",
      "x:  tensor([[10, 11, 12, 13, 14]])\n",
      "y:  tensor([[11, 12, 13, 14, 15]])\n"
     ]
    }
   ],
   "source": [
    "# 随机采样的效果\n",
    "test_seq = list(range(20))\n",
    "print('original_seq: ', test_seq)\n",
    "for x, y in data_iter_random(test_seq, 4, 5):\n",
    "    print('x: ', x)\n",
    "    print('y: ', y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "E6F1E64ABB6C4B22851C80E7BE3DC755",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 第二种：相邻采样\n",
    "def data_iter_consecutive(corpus_indices, batch_size, num_steps):\n",
    "#     if device is None:\n",
    "#         device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    corpus_len = len(corpus_indices) // batch_size * batch_size  # 保留下来的序列的长度\n",
    "    corpus_indices = corpus_indices[: corpus_len]  # 仅保留前corpus_len个字符\n",
    "    indices = torch.tensor(corpus_indices)\n",
    "    indices = indices.view(batch_size, -1)  # resize成(batch_size, )\n",
    "    batch_num = (indices.shape[1] - 1) // num_steps\n",
    "    for i in range(batch_num):\n",
    "        i = i * num_steps\n",
    "        X = indices[:, i: i + num_steps]\n",
    "        Y = indices[:, i + 1: i + num_steps + 1]\n",
    "        yield X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "id": "45B6ACFF081D454D8AD5BEB3542FFE62",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X:  tensor([[ 0,  1,  2,  3,  4,  5],\n",
      "        [15, 16, 17, 18, 19, 20]]) \n",
      "Y: tensor([[ 1,  2,  3,  4,  5,  6],\n",
      "        [16, 17, 18, 19, 20, 21]]) \n",
      "\n",
      "X:  tensor([[ 6,  7,  8,  9, 10, 11],\n",
      "        [21, 22, 23, 24, 25, 26]]) \n",
      "Y: tensor([[ 7,  8,  9, 10, 11, 12],\n",
      "        [22, 23, 24, 25, 26, 27]]) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 相邻采样效果\n",
    "for X, Y in data_iter_consecutive(test_seq, batch_size=2, num_steps=6):\n",
    "    print('X: ', X, '\\nY:', Y, '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "id": "45B6ACFF081D454D8AD5BEB3542FFE62",
    "jupyter": {},
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "32B22B2B72CB4AEFB2E4E8D7F5E429E7",
    "jupyter": {},
    "slideshow": {
     "slide_type": "slide"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}