Improve docstring of base tuner and assessor (#1669)

docs/en_US/Makefile

@@ -16,4 +16,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/en_US/sdk_reference.rst

@@ -41,10 +41,13 @@ Assessor
 ..  autoclass:: nni.assessor.Assessor
     :members:
 
-..  autoclass:: nni.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor
+..  autoclass:: nni.assessor.AssessResult
     :members:
 
-..  autoclass:: nni.medianstop_assessor.medianstop_assessor.MedianstopAssessor
+..  autoclass:: nni.curvefitting_assessor.CurvefittingAssessor
+    :members:
+
+..  autoclass:: nni.medianstop_assessor.MedianstopAssessor
     :members:
 
 
@@ -57,4 +60,4 @@ Advisor
     :members:
 
 ..  autoclass:: nni.bohb_advisor.bohb_advisor.BOHB
-    :members:
+    :members:

src/sdk/pynni/nni/assessor.py

@@ -18,44 +18,118 @@
 # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 # ==================================================================================================
 
+"""
+Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset)
+to tell whether this trial can be early stopped or not.
+
+See :class:`Assessor`' specification and ``docs/en_US/assessors.rst`` for details.
+"""
 
-import logging
 from enum import Enum
+import logging
 
 from .recoverable import Recoverable
 
+__all__ = ['AssessResult', 'Assessor']
+
 _logger = logging.getLogger(__name__)
 
+
 class AssessResult(Enum):
+    """
+    Enum class for :meth:`Assessor.assess_trial` return value.
+    """
+
     Good = True
+    """The trial works well."""
+
     Bad = False
+    """The trial works poorly and should be early stopped."""
+
 
 class Assessor(Recoverable):
+    """
+    Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset)
+    to tell whether this trial can be early stopped or not.
+
+    This is the abstract base class for all assessors.
+    Early stopping algorithms should derive this class and override :meth:`assess_trial` method,
+    which receives intermediate results from trials and give an assessing result.
+
+    If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial,
+    it hints NNI framework that the trial is likely to result in a poor final accuracy,
+    and therefore should be killed to save resource.
+
+    If an accessor want's to get notified when a trial ends, it can also override :meth:`trial_end`.
+
+    To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example.
+
+    See Also
+    --------
+    Builtin assessors:
+    :class:`~nni.medianstop_assessor.MedianstopAssessor`
+    :class:`~nni.curvefitting_assessor.CurvefittingAssessor`
+    """
 
     def assess_trial(self, trial_job_id, trial_history):
-        """Determines whether a trial should be killed. Must override.
-        trial_job_id: identifier of the trial (str).
-        trial_history: a list of intermediate result objects.
-        Returns AssessResult.Good or AssessResult.Bad.
+        """
+        Abstract method for determining whether a trial should be killed. Must override.
+
+        The NNI framework has little guarantee on ``trial_history``.
+        This method is not guaranteed to be invoked for each time ``trial_history`` get updated.
+        It is also possible that a trial's history keeps updateing after receiving a bad result.
+        And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value.
+
+        The only guarantee is that ``trial_history`` is always growing.
+        It will not be empty and will always be longer than previous value.
+
+        This is an example of how :meth:`assess_trial` get invoked sequentially:
+
+        ::
+
+            trial_job_id | trial_history   | return value
+            ------------ | --------------- | ------------
+            Trial_A      | [1.0, 2.0]      | Good
+            Trial_B      | [1.5, 1.3]      | Bad
+            Trial_B      | [1.5, 1.3, 1.9] | Good
+            Trial_A      | [0.9, 1.8, 2.3] | Good
+
+        Parameters
+        ----------
+        trial_job_id: str
+            Unique identifier of the trial.
+        trial_history: list
+            Intermediate results of this trial. The element type is decided by trial code.
+
+        Returns
+        -------
+        AssessResult
+            :obj:`AssessResult.Good` or :obj:`AssessResult.Bad`.
         """
         raise NotImplementedError('Assessor: assess_trial not implemented')
 
     def trial_end(self, trial_job_id, success):
-        """Invoked when a trial is completed or terminated. Do nothing by default.
-        trial_job_id: identifier of the trial (str).
-        success: True if the trial successfully completed; False if failed or terminated.
+        """
+        Abstract method invoked when a trial is completed or terminated. Do nothing by default.
+
+        Parameters
+        ----------
+        trial_job_id: str
+            Unique identifier of the trial.
+        success: bool
+            True if the trial successfully completed; False if failed or terminated.
         """
 
     def load_checkpoint(self):
-        """Load the checkpoint of assessr.
-        path: checkpoint directory for assessor
+        """
+        Internal API under revising, not recommended for end users.
         """
         checkpoin_path = self.get_checkpoint_path()
         _logger.info('Load checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path)
 
     def save_checkpoint(self):
-        """Save the checkpoint of assessor.
-        path: checkpoint directory for assessor
+        """
+        Internal API under revising, not recommended for end users.
         """
         checkpoin_path = self.get_checkpoint_path()
         _logger.info('Save checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path)

src/sdk/pynni/nni/curvefitting_assessor/__init__.py

@@ -0,0 +1 @@
+from .curvefitting_assessor import CurvefittingAssessor

src/sdk/pynni/nni/medianstop_assessor/__init__.py

@@ -0,0 +1 @@
+from .medianstop_assessor import MedianstopAssessor