bigdata-ustc · BAOOOOOM · Aug 22, 2021 · Aug 13, 2021 · Aug 13, 2021 · Aug 13, 2021
diff --git a/AUTHORS.md b/AUTHORS.md
@@ -12,5 +12,6 @@
 
 [Longhu Qin](https://github.com/KenelmQLH)
 
+[Meikai Bao](https://github.com/BAOOOOOM)
 
-The stared contributors are the corresponding authors. 
+The stared contributors are the corresponding authors. 
diff --git a/README.md b/README.md
@@ -41,6 +41,14 @@ pip install EduNLP
 pip install EduNLP[full]
 ```
 
+### Usage
+
+```python
+from EduNLP import get_pretrained_i2v
+i2v = get_pretrained_i2v("d2v_all_256", "./model")
+item_vector, token_vector = i2v(["the content of item 1", "the content of item 2"])
+```
+
 ### Tutorial
 
 For more details, please refer to the full documentation ([latest](https://edunlp.readthedocs.io/en/latest) | [stable](https://edunlp.readthedocs.io/en/stable)).

diff --git a/asset/_static/d2v.png b/asset/_static/d2v.png
diff --git a/asset/_static/d2v_bow_tfidf.png b/asset/_static/d2v_bow_tfidf.png
diff --git a/asset/_static/d2v_general.png b/asset/_static/d2v_general.png
diff --git a/asset/_static/d2v_stem_tf.png b/asset/_static/d2v_stem_tf.png
diff --git a/asset/_static/data.png b/asset/_static/data.png
diff --git a/asset/_static/formula.png b/asset/_static/formula.png
diff --git a/asset/_static/i2v.png b/asset/_static/i2v.png
diff --git a/asset/_static/parse.png b/asset/_static/parse.png
diff --git a/asset/_static/prepare_dataset.jpg b/asset/_static/prepare_dataset.jpg
diff --git a/asset/_static/seg.png b/asset/_static/seg.png
diff --git a/asset/_static/sif.png b/asset/_static/sif.png
diff --git a/asset/_static/sif_addition.png b/asset/_static/sif_addition.png
diff --git a/asset/_static/tokenizer.png b/asset/_static/tokenizer.png
diff --git a/asset/_static/w2v_stem_text.png b/asset/_static/w2v_stem_text.png
diff --git a/asset/_static/w2v_stem_tf.png b/asset/_static/w2v_stem_tf.png
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -2,4 +2,5 @@ sphinx
 sphinx_rtd_theme
 sphinx_toggleprompt
 sphinx-gallery>=0.6
-nbsphinx
+nbsphinx
+m2r2
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -46,14 +46,34 @@ def copy_tree(src, tar):
     'sphinx.ext.mathjax',
     'sphinx_toggleprompt',
     'nbsphinx',
-    'sphinx_gallery.load_style'
+    'sphinx_gallery.load_style',
+    'm2r2',
+    'IPython.sphinxext.ipython_console_highlighting',
+    'IPython.sphinxext.ipython_directive'
 ]
 
 # extension variables setting
 # npsphinx
 
 nbsphinx_thumbnails = {
-    'build/blitz/sif/sif': '_static/item_figure.png',
+    'build/blitz/sif/sif': '_static/sif.png',
+    'build/blitz/sif/sif_addition': '_static/sif_addition.png',
+    'build/blitz/utils/data': '_static/data.png',
+    'build/blitz/formula/formula': '_static/formula.png',
+    'build/blitz/seg/seg': '_static/seg.png',
+    'build/blitz/parse/parse': '_static/parse.png',
+    'build/blitz/formula/formula': '_static/formula.png',
+    'build/blitz/tokenizer/tokenizer': '_static/tokenizer.png',
+    'build/blitz/vectorization/i2v': '_static/i2v.png',
+    'build/blitz/pretrain/prepare_dataset': '_static/prepare_dataset.jpg',
+    'build/blitz/pretrain/gensim/d2v_bow_tfidf': '_static/d2v_bow_tfidf.png',
+    'build/blitz/pretrain/gensim/d2v_general': '_static/d2v_general.png',
+    'build/blitz/pretrain/gensim/d2v_stem_tf': '_static/d2v_stem_tf.png',
+    'build/blitz/pretrain/gensim/w2v_stem_text': '_static/w2v_stem_text.png',
+    'build/blitz/pretrain/gensim/w2v_stem_tf': '_static/w2v_stem_tf.png',
+    'build/blitz/pretrain/seg_token/d2v': '_static/d2v.png',
+    'build/blitz/pretrain/seg_token/d2v_d1': '_static/d2v_d1.png',
+    'build/blitz/pretrain/seg_token/d2v_d2': '_static/d2v_d2.png',
 }
 
 # Add any paths that contain templates here, relative to this directory.
@@ -62,7 +82,7 @@ def copy_tree(src, tar):
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
-source_suffix = ['.rst', '.md', '.ipynb']
+source_suffix = ['.rst', '.md']
 # source_suffix = '.rst'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -75,7 +95,7 @@ def copy_tree(src, tar):
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build']
+exclude_patterns = ['_build','**.ipynb_checkpoints']
 
 # -- Options for HTML output -------------------------------------------------
 

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -84,6 +84,16 @@ But you can also install from source:
 
 Getting Started
 ------------------
+
+One basic usage of EduNLP is to convert an item into a vector, i.e.,
+
+.. code-block:: python
+
+   from EduNLP import get_pretrained_i2v
+   i2v = get_pretrained_i2v("d2v_all_256", "./model")
+   item_vector, token_vector = i2v(["the content of item 1", "the content of item 2"])
+
+
 For absolute beginners, start with the :doc:`Tutorial to EduNLP <tutorial/en/index>` :doc:`(中文版) <tutorial/zh/index>`.
 It covers the basic concepts of EduNLP and
 a step-by-step on training, loading and using the language models.

diff --git a/docs/source/tutorial/zh/index.rst b/docs/source/tutorial/zh/index.rst
@@ -11,13 +11,138 @@
   tokenize
   vectorization
 
-
 示例
 --------
+
+标准项目格式
+^^^^^^^^
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: sif_gallery
+    :glob:
+
+    Code for beginner to learn how to use SIF4Sci  <../../build/blitz/sif/sif>
+    Code for beginner to learn how to use sif_additon  <../../build/blitz/sif/sif_addition>
+
+
+成分分解
+^^^^^^^^^^^
+
+语义成分分解
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: dict2str4sif_gallery
+    :glob:
+
+    Code for beginner to learn how to use dict2str4sif  <../../build/blitz/utils/data.ipynb>
+
+
+结构成分分解
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: seg_gallery
+    :glob:
+
+    Code for beginner to learn how to use seg  <../../build/blitz/seg/seg.ipynb>
+
+
+语法解析
+^^^^^^^^^^^
+
+文本语法结构解析
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: parse_gallery
+    :glob:
+
+    Code for beginner to learn how to use parse  <../../build/blitz/parse/parse.ipynb>
+
+
+公式语法结构解析
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: formula_gallery
+    :glob:
+
+    Code for beginner to learn how to use Formula  <../../build/blitz/formula/formula.ipynb>
+
+
+令牌化
+^^^^^^^^^^^
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: tokenizer_gallery
+    :glob:
+
+    Code for beginner to learn how to use Tokenizer  <../../build/blitz/tokenizer/tokenizer.ipynb>
+
+
+向量化
+^^^^^^^^^^^
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: vectorization_gallery
+    :glob:
+
+    Code for beginner to learn how to use i2v  <../../build/blitz/vectorization/i2v.ipynb>
+
+
+预训练
+^^^^^^^^^^^
+
+获得数据集
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: rst1-gallery
+    :glob:
+
+    prepare_dataset  <../../build/blitz/pretrain/prepare_dataset.ipynb>
+
+
+gensim模型d2v例子
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: rst2-gallery
+    :glob:
+
+    d2v_general  <../../build/blitz/pretrain/gensim/d2v_general.ipynb>
+    d2v_bow_tfidf  <../../build/blitz/pretrain/gensim/d2v_bow_tfidf.ipynb>
+    d2v_stem_tf  <../../build/blitz/pretrain/gensim/d2v_stem_tf.ipynb>
+
+
+gensim模型w2v例子
+####################
+
+.. nbgallery::
+    :caption: This is a thumbnail gallery:
+    :name: rst3-gallery
+    :glob:
+
+    w2v_stem_text  <../../build/blitz/pretrain/gensim/w2v_stem_text.ipynb>
+    w2v_stem_tf  <../../build/blitz/pretrain/gensim/w2v_stem_tf.ipynb>
+
+
+seg_token例子
+####################
+
 .. nbgallery::
     :caption: This is a thumbnail gallery:
-    :name: gallery
+    :name: rst4-gallery
     :glob:
-    :reversed:
 
-    ../../build/blitz/sif/sif
+    d2v.ipynb  <../../build/blitz/pretrain/seg_token/d2v.ipynb>
diff --git a/docs/source/tutorial/zh/parse.rst b/docs/source/tutorial/zh/parse.rst
@@ -6,5 +6,31 @@
 * 文本语法结构解析
 * 公式语法结构解析
 
-公式语法结构解析
+其目的是：
+
+
+1、将选择题中的括号，填空题中的下划线用特殊标识替换掉，并将字符、公式用$$包裹起来，使item能通过$符号准确的按照类型切割开；
+
+2、判断当前item是否合法，并报出错误类型。
+
+具体处理内容
+--------------------
+
+1.匹配公式之外的英文字母、数字，只对两个汉字之间的字母、数字做修正，其余匹配到的情况视为不合 latex 语法录入的公式
+
+2.匹配“（  ）”型括号（包含英文格式和中文格式），即括号内无内容或为空格的括号，将括号替换$\\SIFChoice$
+
+3.匹配下划线，替换连续的下划线或下划线中夹杂空格的情况，将其替换为$\\SIFBlank$
+
+4.匹配latex公式，主要检查latex公式的完整性和可解析性，对latex 中出现中文字符发出警告
+
+学习路线图
 --------------------
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+
+   文本语法结构解析 <parse/文本语法结构解析>
+   公式语法结构解析 <parse/公式语法结构解析>
+
diff --git a/docs/source/tutorial/zh/parse/公式语法结构解析.rst b/docs/source/tutorial/zh/parse/公式语法结构解析.rst
@@ -0,0 +1,61 @@
+公式语法结构解析
+--------------------
+
+本功能主要由EduNLP.Formula模块实现，具有检查传入的公式是否合法，并将合法的公式转换为art树的形式。从实际使用的角度，本模块常作为中间处理过程，调用相应的模型即可自动选择本模块的相关参数，故一般不需要特别关注。
+
+主要内容介绍
++++++++++++++++
+
+1.Formula:对传入的单个公式进行判断，判断传入的公式是否为str形式，如果是则使用ast的方法进行处理，否则进行报错。此外，提供了variable_standardization参数，当此参数为True时，使用变量标准化方法，即同一变量拥有相同的变量编号。
+
+2.FormulaGroup:如果需要传入公式集则可调用此接口，最终将形成ast森林，森林中树的结构同Formula。
+
+
+Examples：
+
+::
+
+ >>> text = '支持公式如$\\frac{y}{x}$，$\\SIFBlank$，$\\FigureID{1}$，不支持公式如$\\frac{ \\dddot y}{x}$'
+ >>> text_parser = Parser(text)
+ >>> text_parser.description_list()
+ >>> text_parser.fomula_illegal_flag
+ >>> 1
+
+::
+
+ >>> f = Formula("x")
+ >>> f
+ <Formula: x>
+ >>> f.ast
+ [{'val': {'id': 0, 'type': 'mathord', 'text': 'x', 'role': None}, 'structure': {'bro': [None, None], 'child': None, 'father': None, 'forest': None}}]
+ >>> f.elements
+ [{'id': 0, 'type': 'mathord', 'text': 'x', 'role': None}]
+ >>> f.variable_standardization(inplace=True)
+ <Formula: x>
+ >>> f.elements
+ [{'id': 0, 'type': 'mathord', 'text': 'x', 'role': None, 'var': 0}]
+
+::
+
+ >>> fg = FormulaGroup(["x + y", "y + x", "z + x"])
+ >>> fg
+ <FormulaGroup: <Formula: x + y>;<Formula: y + x>;<Formula: z + x>>
+ >>> fg = FormulaGroup(["x + y", Formula("y + x"), "z + x"])
+ >>> fg
+ <FormulaGroup: <Formula: x + y>;<Formula: y + x>;<Formula: z + x>>
+ >>> fg = FormulaGroup(["x", Formula("y"), "x"])
+ >>> fg.elements
+ [{'id': 0, 'type': 'mathord', 'text': 'x', 'role': None}, {'id': 1, 'type': 'mathord', 'text': 'y', 'role': None},\
+ {'id': 2, 'type': 'mathord', 'text': 'x', 'role': None}]
+ >>> fg = FormulaGroup(["x", Formula("y"), "x"], variable_standardization=True)
+ >>> fg.elements
+ [{'id': 0, 'type': 'mathord', 'text': 'x', 'role': None, 'var': 0}, {'id': 1, 'type': 'mathord', 'text': 'y', 'role': None, 'var': 1}, {'id': 2, 'type': 'mathord', 'text': 'x', 'role': None, 'var': 0}]
+
+详细示范
++++++++++++++++
+
+.. toctree::
+   :titlesonly:
+
+   树型处理效果 <../../../build/blitz/formula/tree.ipynb>
+   公式解析效果案例 <../../../build/blitz/formula/formula.ipynb>
diff --git a/docs/source/tutorial/zh/parse/文本语法结构解析.rst b/docs/source/tutorial/zh/parse/文本语法结构解析.rst
@@ -0,0 +1,39 @@
+文本语法结构解析
+--------------------
+
+本部分主要由EduNLP.SIF.Parse模块实现，主要功能为将文本中的字母、数字等进行提取，将其转换为标准格式。
+
+主要流程介绍
++++++++++++++++
+
+1.按照以下顺序，先后对传入的文本进行判断类型
+
+* is_chinese：用于匹配中文字符 [\u4e00-\u9fa5]
+
+* is_alphabet：匹配公式之外的英文字母，将匹配到的只对两个汉字之间的字母做修正（使用$$包裹起来），其余匹配到的情况视为不合 latex 语法录入的公式
+
+* is_number：匹配公式之外的数字，只对两个汉字之间的数字做修正（使用$$包裹起来），其余匹配到的情况视为不合 latex 语法录入的公式
+
+2.匹配 latex 公式
+
+* latex 中出现中文字符，打印且只打印一次 warning
+
+* 使用_is_formula_legal函数，检查latex公式的完整性和可解析性，对于不合法公式报错
+
+Examples：
+
+::
+
+ >>> text = '生产某种零件的A工厂25名工人的日加工零件数_   _'
+ >>> text_parser = Parser(text)
+ >>> text_parser.description_list()
+ >>> text_parser.text
+ >>> '生产某种零件的$A$工厂$25$名工人的日加工零件数$\\SIFBlank$'
+
+详细示范
++++++++++++++++
+
+.. toctree::
+   :titlesonly:
+
+   文本语法结构解析的案例 <../../../build/blitz/parse/parse.ipynb>