images/novel_preprocessing.svg

<?xml version="1.0" encoding="UTF-8"?>
<svg width="320px" height="312px" viewBox="0 0 320 312" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
    <!-- Generator: Sketch 46.2 (44496) - http://www.bohemiancoding.com/sketch -->
    <title>Group</title>
    <desc>Created with Sketch.</desc>
    <defs></defs>
    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
        <g id="Group">
            <path d="M139.176471,151.470588 L120.823529,151.470588 C119.264235,151.470588 118,150.206353 118,148.647059 L118,121.823529 C118,120.264235 119.264235,119 120.823529,119 L134.216235,119 C134.217647,119 134.219765,119 134.221176,119 L134.235294,119 L134.235294,119 C134.461176,119 134.653882,119.113647 134.783059,119.278824 L141.721176,126.216941 C141.887059,126.346118 142,126.538824 142,126.764706 L142,126.764706 L142,126.779529 C142,126.780941 142,126.781647 142,126.783059 L142,148.647059 C142,150.206353 140.735765,151.470588 139.176471,151.470588 Z M134.941176,121.386588 L134.941176,126.058824 L139.613412,126.058824 L134.941176,121.386588 Z M140.588235,127.470588 L134.235294,127.470588 C133.844941,127.470588 133.529412,127.154353 133.529412,126.764706 L133.529412,120.411765 L120.823529,120.411765 C120.044235,120.411765 119.411765,121.044235 119.411765,121.823529 L119.411765,148.647059 C119.411765,149.426353 120.044235,150.058824 120.823529,150.058824 L139.176471,150.058824 C139.955765,150.058824 140.588235,149.426353 140.588235,148.647059 L140.588235,127.470588 Z M136.352941,145.823529 L123.647059,145.823529 C123.256706,145.823529 122.941176,145.508 122.941176,145.117647 C122.941176,144.728 123.256706,144.411765 123.647059,144.411765 L136.352941,144.411765 C136.743294,144.411765 137.058824,144.728 137.058824,145.117647 C137.058824,145.508 136.743294,145.823529 136.352941,145.823529 Z M136.352941,140.176471 L123.647059,140.176471 C123.256706,140.176471 122.941176,139.860941 122.941176,139.470588 C122.941176,139.080941 123.256706,138.764706 123.647059,138.764706 L136.352941,138.764706 C136.743294,138.764706 137.058824,139.080941 137.058824,139.470588 C137.058824,139.860941 136.743294,140.176471 136.352941,140.176471 Z M136.352941,134.529412 L123.647059,134.529412 C123.256706,134.529412 122.941176,134.213882 122.941176,133.823529 C122.941176,133.433882 123.256706,133.117647 123.647059,133.117647 L136.352941,133.117647 C136.743294,133.117647 137.058824,133.433882 137.058824,133.823529 C137.058824,134.213882 136.743294,134.529412 136.352941,134.529412 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M137,51 L124,51 C122.8955,51 122,50.1045 122,49 L122,30 C122,28.8955 122.8955,28 124,28 L133.4865,28 C133.4875,28 133.489,28 133.49,28 L133.5,28 L133.5,28 C133.66,28 133.7965,28.0805 133.888,28.1975 L138.8025,33.112 C138.92,33.2035 139,33.34 139,33.5 L139,33.5 L139,33.5105 C139,33.5115 139,33.512 139,33.513 L139,49 C139,50.1045 138.1045,51 137,51 Z M134,29.6905 L134,33 L137.3095,33 L134,29.6905 Z M138,34 L133.5,34 C133.2235,34 133,33.776 133,33.5 L133,29 L124,29 C123.448,29 123,29.448 123,30 L123,49 C123,49.552 123.448,50 124,50 L137,50 C137.552,50 138,49.552 138,49 L138,34 Z M135,47 L126,47 C125.7235,47 125.5,46.7765 125.5,46.5 C125.5,46.224 125.7235,46 126,46 L135,46 C135.2765,46 135.5,46.224 135.5,46.5 C135.5,46.7765 135.2765,47 135,47 Z M135,43 L126,43 C125.7235,43 125.5,42.7765 125.5,42.5 C125.5,42.224 125.7235,42 126,42 L135,42 C135.2765,42 135.5,42.224 135.5,42.5 C135.5,42.7765 135.2765,43 135,43 Z M135,39 L126,39 C125.7235,39 125.5,38.7765 125.5,38.5 C125.5,38.224 125.7235,38 126,38 L135,38 C135.2765,38 135.5,38.224 135.5,38.5 C135.5,38.7765 135.2765,39 135,39 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M147,77 L134,77 C132.8955,77 132,76.1045 132,75 L132,56 C132,54.8955 132.8955,54 134,54 L143.4865,54 C143.4875,54 143.489,54 143.49,54 L143.5,54 L143.5,54 C143.66,54 143.7965,54.0805 143.888,54.1975 L148.8025,59.112 C148.92,59.2035 149,59.34 149,59.5 L149,59.5 L149,59.5105 C149,59.5115 149,59.512 149,59.513 L149,75 C149,76.1045 148.1045,77 147,77 Z M144,55.6905 L144,59 L147.3095,59 L144,55.6905 Z M148,60 L143.5,60 C143.2235,60 143,59.776 143,59.5 L143,55 L134,55 C133.448,55 133,55.448 133,56 L133,75 C133,75.552 133.448,76 134,76 L147,76 C147.552,76 148,75.552 148,75 L148,60 Z M145,73 L136,73 C135.7235,73 135.5,72.7765 135.5,72.5 C135.5,72.224 135.7235,72 136,72 L145,72 C145.2765,72 145.5,72.224 145.5,72.5 C145.5,72.7765 145.2765,73 145,73 Z M145,69 L136,69 C135.7235,69 135.5,68.7765 135.5,68.5 C135.5,68.224 135.7235,68 136,68 L145,68 C145.2765,68 145.5,68.224 145.5,68.5 C145.5,68.7765 145.2765,69 145,69 Z M145,65 L136,65 C135.7235,65 135.5,64.7765 135.5,64.5 C135.5,64.224 135.7235,64 136,64 L145,64 C145.2765,64 145.5,64.224 145.5,64.5 C145.5,64.7765 145.2765,65 145,65 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M126,74 L113,74 C111.8955,74 111,73.1045 111,72 L111,53 C111,51.8955 111.8955,51 113,51 L122.4865,51 C122.4875,51 122.489,51 122.49,51 L122.5,51 L122.5,51 C122.66,51 122.7965,51.0805 122.888,51.1975 L127.8025,56.112 C127.92,56.2035 128,56.34 128,56.5 L128,56.5 L128,56.5105 C128,56.5115 128,56.512 128,56.513 L128,72 C128,73.1045 127.1045,74 126,74 Z M123,52.6905 L123,56 L126.3095,56 L123,52.6905 Z M127,57 L122.5,57 C122.2235,57 122,56.776 122,56.5 L122,52 L113,52 C112.448,52 112,52.448 112,53 L112,72 C112,72.552 112.448,73 113,73 L126,73 C126.552,73 127,72.552 127,72 L127,57 Z M124,70 L115,70 C114.7235,70 114.5,69.7765 114.5,69.5 C114.5,69.224 114.7235,69 115,69 L124,69 C124.2765,69 124.5,69.224 124.5,69.5 C124.5,69.7765 124.2765,70 124,70 Z M124,66 L115,66 C114.7235,66 114.5,65.7765 114.5,65.5 C114.5,65.224 114.7235,65 115,65 L124,65 C124.2765,65 124.5,65.224 124.5,65.5 C124.5,65.7765 124.2765,66 124,66 Z M124,62 L115,62 C114.7235,62 114.5,61.7765 114.5,61.5 C114.5,61.224 114.7235,61 115,61 L124,61 C124.2765,61 124.5,61.224 124.5,61.5 C124.5,61.7765 124.2765,62 124,62 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M157,52 L144,52 C142.8955,52 142,51.1045 142,50 L142,31 C142,29.8955 142.8955,29 144,29 L153.4865,29 C153.4875,29 153.489,29 153.49,29 L153.5,29 L153.5,29 C153.66,29 153.7965,29.0805 153.888,29.1975 L158.8025,34.112 C158.92,34.2035 159,34.34 159,34.5 L159,34.5 L159,34.5105 C159,34.5115 159,34.512 159,34.513 L159,50 C159,51.1045 158.1045,52 157,52 Z M154,30.6905 L154,34 L157.3095,34 L154,30.6905 Z M158,35 L153.5,35 C153.2235,35 153,34.776 153,34.5 L153,30 L144,30 C143.448,30 143,30.448 143,31 L143,50 C143,50.552 143.448,51 144,51 L157,51 C157.552,51 158,50.552 158,50 L158,35 Z M155,48 L146,48 C145.7235,48 145.5,47.7765 145.5,47.5 C145.5,47.224 145.7235,47 146,47 L155,47 C155.2765,47 155.5,47.224 155.5,47.5 C155.5,47.7765 155.2765,48 155,48 Z M155,44 L146,44 C145.7235,44 145.5,43.7765 145.5,43.5 C145.5,43.224 145.7235,43 146,43 L155,43 C155.2765,43 155.5,43.224 155.5,43.5 C155.5,43.7765 155.2765,44 155,44 Z M155,40 L146,40 C145.7235,40 145.5,39.7765 145.5,39.5 C145.5,39.224 145.7235,39 146,39 L155,39 C155.2765,39 155.5,39.224 155.5,39.5 C155.5,39.7765 155.2765,40 155,40 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M117,47 L104,47 C102.8955,47 102,46.1045 102,45 L102,26 C102,24.8955 102.8955,24 104,24 L113.4865,24 C113.4875,24 113.489,24 113.49,24 L113.5,24 L113.5,24 C113.66,24 113.7965,24.0805 113.888,24.1975 L118.8025,29.112 C118.92,29.2035 119,29.34 119,29.5 L119,29.5 L119,29.5105 C119,29.5115 119,29.512 119,29.513 L119,45 C119,46.1045 118.1045,47 117,47 Z M114,25.6905 L114,29 L117.3095,29 L114,25.6905 Z M118,30 L113.5,30 C113.2235,30 113,29.776 113,29.5 L113,25 L104,25 C103.448,25 103,25.448 103,26 L103,45 C103,45.552 103.448,46 104,46 L117,46 C117.552,46 118,45.552 118,45 L118,30 Z M115,43 L106,43 C105.7235,43 105.5,42.7765 105.5,42.5 C105.5,42.224 105.7235,42 106,42 L115,42 C115.2765,42 115.5,42.224 115.5,42.5 C115.5,42.7765 115.2765,43 115,43 Z M115,39 L106,39 C105.7235,39 105.5,38.7765 105.5,38.5 C105.5,38.224 105.7235,38 106,38 L115,38 C115.2765,38 115.5,38.224 115.5,38.5 C115.5,38.7765 115.2765,39 115,39 Z M115,35 L106,35 C105.7235,35 105.5,34.7765 105.5,34.5 C105.5,34.224 105.7235,34 106,34 L115,34 C115.2765,34 115.5,34.224 115.5,34.5 C115.5,34.7765 115.2765,35 115,35 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M149,24 L136,24 C134.8955,24 134,23.1045 134,22 L134,3 C134,1.8955 134.8955,1 136,1 L145.4865,1 C145.4875,1 145.489,1 145.49,1 L145.5,1 L145.5,1 C145.66,1 145.7965,1.0805 145.888,1.1975 L150.8025,6.112 C150.92,6.2035 151,6.34 151,6.5 L151,6.5 L151,6.5105 C151,6.5115 151,6.512 151,6.513 L151,22 C151,23.1045 150.1045,24 149,24 Z M146,2.6905 L146,6 L149.3095,6 L146,2.6905 Z M150,7 L145.5,7 C145.2235,7 145,6.776 145,6.5 L145,2 L136,2 C135.448,2 135,2.448 135,3 L135,22 C135,22.552 135.448,23 136,23 L149,23 C149.552,23 150,22.552 150,22 L150,7 Z M147,20 L138,20 C137.7235,20 137.5,19.7765 137.5,19.5 C137.5,19.224 137.7235,19 138,19 L147,19 C147.2765,19 147.5,19.224 147.5,19.5 C147.5,19.7765 147.2765,20 147,20 Z M147,16 L138,16 C137.7235,16 137.5,15.7765 137.5,15.5 C137.5,15.224 137.7235,15 138,15 L147,15 C147.2765,15 147.5,15.224 147.5,15.5 C147.5,15.7765 147.2765,16 147,16 Z M147,12 L138,12 C137.7235,12 137.5,11.7765 137.5,11.5 C137.5,11.224 137.7235,11 138,11 L147,11 C147.2765,11 147.5,11.224 147.5,11.5 C147.5,11.7765 147.2765,12 147,12 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <path d="M127,23 L114,23 C112.8955,23 112,22.1045 112,21 L112,2 C112,0.8955 112.8955,0 114,0 L123.4865,0 C123.4875,0 123.489,0 123.49,0 L123.5,0 L123.5,0 C123.66,0 123.7965,0.0805 123.888,0.1975 L128.8025,5.112 C128.92,5.2035 129,5.34 129,5.5 L129,5.5 L129,5.5105 C129,5.5115 129,5.512 129,5.513 L129,21 C129,22.1045 128.1045,23 127,23 Z M124,1.6905 L124,5 L127.3095,5 L124,1.6905 Z M128,6 L123.5,6 C123.2235,6 123,5.776 123,5.5 L123,1 L114,1 C113.448,1 113,1.448 113,2 L113,21 C113,21.552 113.448,22 114,22 L127,22 C127.552,22 128,21.552 128,21 L128,6 Z M125,19 L116,19 C115.7235,19 115.5,18.7765 115.5,18.5 C115.5,18.224 115.7235,18 116,18 L125,18 C125.2765,18 125.5,18.224 125.5,18.5 C125.5,18.7765 125.2765,19 125,19 Z M125,15 L116,15 C115.7235,15 115.5,14.7765 115.5,14.5 C115.5,14.224 115.7235,14 116,14 L125,14 C125.2765,14 125.5,14.224 125.5,14.5 C125.5,14.7765 125.2765,15 125,15 Z M125,11 L116,11 C115.7235,11 115.5,10.7765 115.5,10.5 C115.5,10.224 115.7235,10 116,10 L125,10 C125.2765,10 125.5,10.224 125.5,10.5 C125.5,10.7765 125.2765,11 125,11 Z" id="Shape" fill="#000000" fill-rule="nonzero"></path>
            <text id="['h','e','l','l','o'" font-family="Helvetica-Bold, Helvetica" font-size="11" font-weight="bold" fill="#000000">
                <tspan x="4" y="200">['h','e','l','l','o',' ',</tspan>
                <tspan x="86.1132812" y="200" font-family="PingFangSC-Semibold, PingFang SC" font-weight="500">'m','y',' ','n','a','m','e',' ','i','s', ...]</tspan>
            </text>
            <text id="[10,-5,-12,-12,-17," font-family="Helvetica-Bold, Helvetica" font-size="10" font-weight="bold" fill="#000000">
                <tspan x="0" y="257">[10, 5, 12, 12, 17, 27, 15, 25, 27, 16, 1, 15, 5, 27, 6, 18, ...]</tspan>
            </text>
            <g id="Arrow-04" transform="translate(130.000000, 99.000000) rotate(90.000000) translate(-130.000000, -99.000000) translate(113.000000, 90.000000)">
                <g>
                    <path d="M0,9 L26,9" id="Stroke-04" stroke="#455A64" stroke-width="2"></path>
                    <path d="M25.3327171,9 C23.0720357,6.93069307 21.9946966,2.83861386 21.906,0 C25.0060541,3.81188119 29.0330957,7.13564356 33.906,9 C29.0330957,10.7851485 25.1834473,14.309901 21.906,18 C22.1720898,14.9594059 22.9833391,11.270297 25.3327171,9" id="Arrow-head-04" fill="#455A64"></path>
                </g>
            </g>
            <g id="Arrow-04" transform="translate(130.000000, 171.000000) rotate(90.000000) translate(-130.000000, -171.000000) translate(113.000000, 162.000000)">
                <g>
                    <path d="M0,9 L26,9" id="Stroke-04" stroke="#455A64" stroke-width="2"></path>
                    <path d="M25.3327171,9 C23.0720357,6.93069307 21.9946966,2.83861386 21.906,0 C25.0060541,3.81188119 29.0330957,7.13564356 33.906,9 C29.0330957,10.7851485 25.1834473,14.309901 21.906,18 C22.1720898,14.9594059 22.9833391,11.270297 25.3327171,9" id="Arrow-head-04" fill="#455A64"></path>
                </g>
            </g>
            <g id="Arrow-04" transform="translate(130.000000, 226.000000) rotate(90.000000) translate(-130.000000, -226.000000) translate(113.000000, 217.000000)">
                <g>
                    <path d="M0,9 L26,9" id="Stroke-04" stroke="#455A64" stroke-width="2"></path>
                    <path d="M25.3327171,9 C23.0720357,6.93069307 21.9946966,2.83861386 21.906,0 C25.0060541,3.81188119 29.0330957,7.13564356 33.906,9 C29.0330957,10.7851485 25.1834473,14.309901 21.906,18 C22.1720898,14.9594059 22.9833391,11.270297 25.3327171,9" id="Arrow-head-04" fill="#455A64"></path>
                </g>
            </g>
            <g id="Arrow-04" transform="translate(130.000000, 279.000000) rotate(90.000000) translate(-130.000000, -279.000000) translate(113.000000, 270.000000)">
                <g>
                    <path d="M0,9 L26,9" id="Stroke-04" stroke="#455A64" stroke-width="2"></path>
                    <path d="M25.3327171,9 C23.0720357,6.93069307 21.9946966,2.83861386 21.906,0 C25.0060541,3.81188119 29.0330957,7.13564356 33.906,9 C29.0330957,10.7851485 25.1834473,14.309901 21.906,18 C22.1720898,14.9594059 22.9833391,11.270297 25.3327171,9" id="Arrow-head-04" fill="#455A64"></path>
                </g>
            </g>
            <text id="merge-all-training-d" font-family="Helvetica" font-size="10" font-weight="normal" fill="#000000">
                <tspan x="153" y="98">merge all training documents into one</tspan>
            </text>
            <text id="load-as-a-flat-list" font-family="Helvetica" font-size="10" font-weight="normal" fill="#000000">
                <tspan x="153" y="169">load as a flat list of chars</tspan>
            </text>
            <text id="convert-chars-to-int" font-family="Helvetica" font-size="10" font-weight="normal" fill="#000000">
                <tspan x="153" y="226">convert chars to integers</tspan>
            </text>
            <text id="use-integers-to-gene" font-family="Helvetica" font-size="10" font-weight="normal" fill="#000000">
                <tspan x="3" y="309">use integers to generate one-hot inputs for each time step</tspan>
            </text>
        </g>
    </g>
</svg>