Refactor import paths (#1358)

* Refactor import paths

* Reduce a few more paths

* RTD improvements (#1359)

* RTD improvements

* Add package source files

* Update reqs

* Update build

* Update packages

* Fix slicing import name

.gitignore

@@ -68,7 +68,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
-docs/source/
+docs/packages/_autosummary
 
 # PyBuilder
 target/

docs/conf.py

@@ -10,6 +10,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 
+import datetime
 import os
 import sys
 
@@ -21,9 +22,10 @@
 # -- Project information -----------------------------------------------------
 
 project = "Snorkel"
-copyright = "2019, Snorkel Team"
+copyright = f"{datetime.datetime.now().year}, Snorkel Team"
 author = "Snorkel Team"
 master_doc = "index"
+html_logo = "_static/octopus.png"
 
 VERSION = {}
 with open("../snorkel/version.py", "r") as version_file:
@@ -36,20 +38,14 @@
 # -- General configuration ---------------------------------------------------
 
 # Mock imports for troublesome modules (i.e. any that use C code)
-autodoc_mock_imports = [
-    "dask",
-    "dask.distributed",
-    "pyspark",
-    "pyspark.sql",
-    "spacy",
-    "tensorboardX",
-]
+autosummary_mock_imports = ["dask", "pyspark", "spacy"]
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
     "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
     "sphinx.ext.napoleon",
     "sphinx_autodoc_typehints",
     "sphinx.ext.linkcode",
@@ -63,6 +59,7 @@
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
+autosummary_generate = True
 
 # -- Options for HTML output -------------------------------------------------
 
@@ -98,7 +95,7 @@
 # directive
 #
 # http://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoclass
-autoclass_content = "both"
+autoclass_content = "class"
 
 
 # Default options to an ..autoXXX directive.
@@ -123,14 +120,6 @@ def linkcode_resolve(domain, info):
     return f"https://github.com/HazyResearch/snorkel/blob/redux/{filename}.py"
 
 
-# -- Run apidoc --------------------------------------------------------------
-def run_apidoc(_):
-    args = ["-f", "-o", "./source/", "../snorkel"]
-    from sphinx.ext import apidoc
-
-    apidoc.main(args)
-
-
 # -- Exclude PyTorch methods -------------------------------------------------
 def skip_torch_module_member(app, what, name, obj, skip, options):
     skip_torch = "Module." in str(obj) and name in dir(torch.nn.Module)
@@ -142,4 +131,3 @@ def skip_torch_module_member(app, what, name, obj, skip, options):
 # -- Run setup ---------------------------------------------------------------
 def setup(app):
     app.connect("autodoc-skip-member", skip_torch_module_member)
-    app.connect("builder-inited", run_apidoc)

docs/index.rst

@@ -10,21 +10,18 @@ If you're looking for technical details on Snorkel's API,
 you're in the right place.
 
 For more narrative walkthroughs of Snorkel fundamentals or
-example use cases, check out our `homepage <http://snorkel.stanford.edu/>`_
+example use cases, check out our `homepage <http://snorkel.org/>`_
 and our `tutorials repo <https://github.com/HazyResearch/snorkel/>`_.
 
-
 .. toctree::
-    :titlesonly:
-    :maxdepth: 2
-    :caption: Package Reference
+   :maxdepth: 1
+   :caption: Package Reference
 
-    source/snorkel.analysis
-    source/snorkel.augmentation
-    source/snorkel.classification
-    source/snorkel.contrib
-    source/snorkel.labeling
-    source/snorkel.map
-    source/snorkel.slicing
-    source/snorkel.synthetic
-    source/snorkel.types
+   packages/analysis
+   packages/augmentation
+   packages/classification
+   packages/labeling
+   packages/map
+   packages/preprocess
+   packages/slicing
+   packages/utils

docs/packages.json

@@ -0,0 +1,33 @@
+{
+    "packages": [
+        "analysis",
+        "augmentation",
+        "classification",
+        "labeling",
+        "map",
+        "preprocess",
+        "slicing",
+        "utils"
+    ],
+    "extra_members": {
+        "labeling": [
+            "apply.dask.DaskLFApplier",
+            "apply.dask.PandasParallelLFApplier",
+            "apply.spark.SparkLFApplier",
+            "lf.nlp.NLPLabelingFunction",
+            "lf.nlp.nlp_labeling_function"
+        ],
+        "map": [
+            "spark.make_spark_mapper"
+        ],
+        "preprocess": [
+            "nlp.SpacyPreprocessor",
+            "spark.make_spark_preprocessor"
+        ],
+        "slicing": [
+            "apply.dask.DaskSFApplier",
+            "apply.dask.PandasParallelSFApplier",
+            "apply.spark.SparkSFApplier"
+        ]
+    }
+}

docs/packages/analysis.rst

@@ -0,0 +1,13 @@
+Snorkel Analysis Package
+------------------------
+
+Generic model analysis utilities shared across Snorkel.
+
+.. currentmodule:: snorkel.analysis
+
+.. autosummary::
+   :toctree: _autosummary/analysis/
+   :nosignatures:
+
+   get_label_buckets
+   metric_score

docs/packages/augmentation.rst

@@ -0,0 +1,20 @@
+Snorkel Augmentation Package
+----------------------------
+
+Programmatic data set augmentation: TF creation and data generation utilities.
+
+.. currentmodule:: snorkel.augmentation
+
+.. autosummary::
+   :toctree: _autosummary/augmentation/
+   :nosignatures:
+
+   ApplyAllPolicy
+   ApplyEachPolicy
+   ApplyOnePolicy
+   MeanFieldPolicy
+   PandasTFApplier
+   RandomPolicy
+   TFApplier
+   TransformationFunction
+   transformation_function

docs/packages/classification.rst

@@ -0,0 +1,27 @@
+Snorkel Classification Package
+------------------------------
+
+PyTorch-based multi-task learning framework for Snorkel-generated datasets.
+
+.. currentmodule:: snorkel.classification
+
+.. autosummary::
+   :toctree: _autosummary/classification/
+   :nosignatures:
+
+   Checkpointer
+   CheckpointerConfig
+   DictDataLoader
+   DictDataset
+   LogManager
+   LogManagerConfig
+   LogWriter
+   LogWriterConfig
+   Operation
+   Scorer
+   SnorkelClassifier
+   Task
+   TensorBoardWriter
+   Trainer
+   ce_loss
+   softmax

docs/packages/labeling.rst

@@ -0,0 +1,26 @@
+Snorkel Labeling Package
+------------------------
+
+Programmatic data set labeling: LF creation, models, and analysis utilities.
+
+.. currentmodule:: snorkel.labeling
+
+.. autosummary::
+   :toctree: _autosummary/labeling/
+   :nosignatures:
+
+   apply.dask.DaskLFApplier
+   LFAnalysis
+   LFApplier
+   LabelModel
+   LabelingFunction
+   MajorityClassVoter
+   MajorityLabelVoter
+   lf.nlp.NLPLabelingFunction
+   PandasLFApplier
+   apply.dask.PandasParallelLFApplier
+   RandomVoter
+   apply.spark.SparkLFApplier
+   filter_unlabeled_dataframe
+   labeling_function
+   lf.nlp.nlp_labeling_function

docs/packages/map.rst

@@ -0,0 +1,16 @@
+Snorkel Map Package
+-------------------
+
+Generic utilities for data point to data point operations.
+
+.. currentmodule:: snorkel.map
+
+.. autosummary::
+   :toctree: _autosummary/map/
+   :nosignatures:
+
+   BaseMapper
+   LambdaMapper
+   Mapper
+   lambda_mapper
+   spark.make_spark_mapper

docs/packages/preprocess.rst

@@ -0,0 +1,17 @@
+Snorkel Preprocess Package
+--------------------------
+
+Preprocessors for LFs, TFs, and SFs.
+
+.. currentmodule:: snorkel.preprocess
+
+.. autosummary::
+   :toctree: _autosummary/preprocess/
+   :nosignatures:
+
+   BasePreprocessor
+   LambdaPreprocessor
+   Preprocessor
+   nlp.SpacyPreprocessor
+   spark.make_spark_preprocessor
+   preprocessor

docs/packages/slicing.rst

@@ -0,0 +1,21 @@
+Snorkel Slicing Package
+-----------------------
+
+Programmatic data set slicing: SF creation, models, and monitoring utilities.
+
+.. currentmodule:: snorkel.slicing
+
+.. autosummary::
+   :toctree: _autosummary/slicing/
+   :nosignatures:
+
+   apply.dask.DaskSFApplier
+   apply.dask.PandasParallelSFApplier
+   PandasSFApplier
+   SFApplier
+   SliceCombinerModule
+   SlicingFunction
+   apply.spark.SparkSFApplier
+   add_slice_labels
+   convert_to_slice_tasks
+   slicing_function

docs/packages/utils.rst

@@ -0,0 +1,16 @@
+Snorkel Utils Package
+---------------------
+
+None
+
+.. currentmodule:: snorkel.utils
+
+.. autosummary::
+   :toctree: _autosummary/utils/
+   :nosignatures:
+
+   filter_labels
+   preds_to_probs
+   probs_to_preds
+   set_seed
+   to_int_label_array

docs/requirements-doc.txt

@@ -1,4 +1,4 @@
-sphinx==1.8.5
-sphinx_autodoc_typehints==1.6.0
+sphinx==2.1.2
+sphinx_autodoc_typehints==1.7.0
 sphinx_rtd_theme==0.4.3
 https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl

requirements.txt

@@ -23,15 +23,14 @@ torch>=1.1.0,<2.0.0
 # LF dependency learning
 networkx>=2.2,<3.0
 
+# Model introspection tools
+tensorboardX>=1.6,<2.0
+
 
 #### EXTRA/TEST LIBRARIES
 
 # NLP applications
 spacy>=2.1.0,<3.0.0
-torchtext>=0.3.0,<0.4.0
-
-# Model introspection tools
-tensorboardX>=1.6,<2.0
 
 # Scale-up/out
 dask[complete]>=2.1.0,<3.0.0