Update Mars on Ray doc (#3311)

* Update Mars doc * Compatible with sphinx 6 * Unpin sphinx<6 * Refine doc * Refine Co-authored-by: 刘宝 <po.lb@antgroup.com>
mars-project · Jan 10, 2023 · dcc090d · dcc090d
1 parent aa1b261
commit dcc090d
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 83 deletions.
diff --git a/README.rst b/README.rst
@@ -244,20 +244,9 @@ Starting a new Mars on Ray runtime locally via:
 
 .. code-block:: python
 
-    import ray
-    ray.init()
     import mars
-    mars.new_ray_session(worker_num=2)
-    import mars.tensor as mt
-    mt.random.RandomState(0).rand(1000_0000, 5).sum().execute()
-
-Or connecting to a Mars on Ray runtime which is already initialized.
-
-.. code-block:: python
-
-    import mars
-    mars.new_ray_session('http://<web_ip>:<ui_port>')
-    # perform computation
+    mars.new_session(backend='ray')
+    # Perform compute
 
 Interact with Ray Dataset:
 

diff --git a/docs/requirements-doc.txt b/docs/requirements-doc.txt
@@ -10,7 +10,7 @@ pytest-cov>=2.5.0
 pytest-timeout>=1.2.0
 cloudpickle>=1.0.0
 sqlalchemy>=1.2.0
-sphinx<6.0.0
+sphinx>=3.0.0
 pydata-sphinx-theme>=0.3.0
 sphinx-intl>=0.9.9
 ipython>=4.0

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -20,9 +20,9 @@
 # -- Project information -----------------------------------------------------
 from mars import __version__
 
-project = 'mars'
-copyright = '1999-2020, The Alibaba Group Holding Ltd.'
-author = 'jisheng, qinxing'
+project = "mars"
+copyright = "1999-2020, The Alibaba Group Holding Ltd."
+author = "jisheng, qinxing"
 
 # The short X.Y version
 version = __version__
@@ -40,29 +40,29 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.mathjax',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.githubpages',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.napoleon',
-    'IPython.sphinxext.ipython_directive',
-    'IPython.sphinxext.ipython_console_highlighting',
-    'matplotlib.sphinxext.plot_directive',
+    "sphinx.ext.mathjax",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.githubpages",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "IPython.sphinxext.ipython_directive",
+    "IPython.sphinxext.ipython_console_highlighting",
+    "matplotlib.sphinxext.plot_directive",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -77,16 +77,16 @@
 exclude_patterns = []
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'pydata_sphinx_theme'
-html_logo = 'images/mars.svg'
+html_theme = "pydata_sphinx_theme"
+html_logo = "images/mars.svg"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -96,14 +96,21 @@
     "github_url": "https://github.com/mars-project/mars",
     "twitter_url": "https://twitter.com/pymars_dev",
     "external_links": [
-        {"name": "Release Notes", "url": "https://github.com/mars-project/mars/releases"},
-    ]
+        {
+            "name": "Release Notes",
+            "url": "https://github.com/mars-project/mars/releases",
+        },
+    ],
+    "logo": {
+        "image_light": "mars.svg",
+        "image_dark": "mars.svg",
+    },
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['images']
+html_static_path = ["images"]
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -119,7 +126,7 @@
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'marsdoc'
+htmlhelp_basename = "marsdoc"
 
 
 # -- Options for LaTeX output ------------------------------------------------
@@ -128,15 +135,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -146,19 +150,15 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'mars.tex', 'mars Documentation',
-     'jisheng,qinxing', 'manual'),
+    (master_doc, "mars.tex", "mars Documentation", "jisheng,qinxing", "manual"),
 ]
 
 
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'mars', 'mars Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, "mars", "mars Documentation", [author], 1)]
 
 
 # -- Options for Texinfo output ----------------------------------------------
@@ -167,9 +167,15 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'mars', 'mars Documentation',
-     author, 'mars', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "mars",
+        "mars Documentation",
+        author,
+        "mars",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 autosummary_generate = True
@@ -191,8 +197,8 @@
 
 # -- Extension configuration -------------------------------------------------
 
-locale_dirs = ['locale/']   # path is example but recommended.
-gettext_compact = False     # optional.
+locale_dirs = ["locale/"]  # path is example but recommended.
+gettext_compact = False  # optional.
 ipython_warning_is_error = False
 
 

diff --git a/docs/source/installation/ray.rst b/docs/source/installation/ray.rst
@@ -11,21 +11,21 @@ Install Ray locally:
 
 .. code-block:: bash
 
-    pip install ray>=1.8.0
+    pip install ray
 
-Start a Ray cluster:
+(Optional) Start a Ray cluster or Mars starts a Ray cluster automatically:
 
 .. code-block:: python
 
     import ray
     ray.init()
 
-Or connecting to a existing Ray cluster using `Ray client <https://docs.ray.io/en/latest/cluster/ray-client.html>`_:
+(Optional) Or connecting to a existing Ray cluster using `Ray client <https://docs.ray.io/en/latest/cluster/ray-client.html>`_:
 
 .. code-block:: python
 
     import ray
-    ray.init(address="ray://<head_node_host>:10001")
+    ray.init(address='ray://<head_node_host>:10001')
 
 Creating a Mars on Ray runtime in the Ray cluster and do the computing:
 
@@ -34,7 +34,8 @@ Creating a Mars on Ray runtime in the Ray cluster and do the computing:
     import mars
     import mars.tensor as mt
     import mars.dataframe as md
-    session = mars.new_ray_session(worker_num=2, worker_mem=2 * 1024 ** 3)
+    # This driver is the Mars supervisor.
+    session = mars.new_session(backend='ray')
     mt.random.RandomState(0).rand(1000_0000, 5).sum().execute()
     df = md.DataFrame(
         mt.random.rand(1000_0000, 4, chunk_size=500_0000),
@@ -44,66 +45,82 @@ Creating a Mars on Ray runtime in the Ray cluster and do the computing:
     # Convert mars dataframe to ray dataset
     ds = md.to_ray_dataset(df)
     print(ds.schema(), ds.count())
-    ds.filter(lambda row: row["a"] > 0.5).show(5)
+    ds.filter(lambda row: row['a'] > 0.5).show(5)
     # Convert ray dataset to mars dataframe
     df2 = md.read_ray_dataset(ds)
     print(df2.head(5).execute())
 
 
-Create a Mars on Ray runtime independently in the Ray cluster:
+Stop the created Mars on Ray runtime:
 
 .. code-block:: python
 
-    import mars
-    import mars.tensor as mt
-    cluster = mars.new_cluster_in_ray(worker_num=2, worker_mem=2 * 1024 ** 3)
+    session.stop_server()
+
+
+Customizing cluster
+-------------------
 
-Connect to the created Mars on Ray runtime and do the computing:
+There are two ways to initialize a Mars on Ray session:
+
+- `mars.new_session(...) # Start Mars supervisor in current process.`
+    Recommend for most use cases.
+- `mars.new_ray_session(...) # Start a Ray actor for Mars supervisor.`
+    Recommend for large scale compute or compute through Ray client.
+
+
+Start a Ray actor for Mars supervisor:
 
 .. code-block:: python
 
     import mars
-    import mars.tensor as mt
-    session = mars.new_ray_session(address="http://ip:port", session_id="abcd", default=True)
-    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    # Start a Ray actor for Mars supervisor.
+    session = mars.new_ray_session(backend='ray')
 
-Stop the created Mars on Ray runtime:
+Connect to the created Mars on Ray runtime and do the computing, the supervisor virtual address is the name of Ray actor for Mars supervisor,
+e.g. `ray://ray-cluster-1672904753/0/0`.
 
 .. code-block:: python
 
-    cluster.stop()
-
+    import mars
+    import mars.tensor as mt
+    # Be aware that `mars.new_ray_session()` connects to an existing Mars
+    # cluster requires Ray runtime.
+    # e.g. Current process is a initialized Ray driver, client or worker.
+    session = mars.new_ray_session(
+        address='ray://<supervisor virtual address>',
+        session_id='abcd',
+        backend='ray',
+        default=True)
+    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
 
-Customizing cluster
--------------------
-``new_ray_session``/``new_cluster_in_ray`` function provides several keyword arguments for users to define
+The ``new_ray_session`` function provides several keyword arguments for users to define
 the cluster.
 
 Arguments for supervisors:
 
-+----------------------+-----------------------------------------------------------+
-| Argument             | Description                                               |
-+======================+===========================================================+
-| supervisor_mem       | Memory size for supervisor in the cluster, in bytes.      |
-+----------------------+-----------------------------------------------------------+
++--------------------+-----------------------------------------------------------------+
+| Argument           | Description                                                     |
++====================+=================================================================+
+| supervisor_cpu     | Number of CPUs for supervisor, 1 by default.                    |
++--------------------+-----------------------------------------------------------------+
+| supervisor_mem     | Memory size for supervisor in bytes, 1G by default.             |
++--------------------+-----------------------------------------------------------------+
 
 Arguments for workers:
 
 +--------------------+-----------------------------------------------------------------+
 | Argument           | Description                                                     |
 +====================+=================================================================+
-| worker_num         | Number of workers in the cluster, 1 by default.                 |
-+--------------------+-----------------------------------------------------------------+
 | worker_cpu         | Number of CPUs for every worker, 2 by default.                  |
 +--------------------+-----------------------------------------------------------------+
-| worker_mem         | Memory size for workers in the cluster, in bytes, 2G by default.|
+| worker_mem         | Memory size for workers in bytes, 2G by default.                |
 +--------------------+-----------------------------------------------------------------+
 
-For instance, if you want to create a Mars cluster with 100 workers,
-each worker has 4 cores and 16GB memory, you can use the code below:
+For instance, if you want to create a Mars cluster with a standalone supervisor,
+you can use the code below (In this example, one Ray node has 16 CPUs in total):
 
 .. code-block:: python
 
     import mars
-    import mars.tensor as mt
-    cluster = mars.new_cluster_in_ray(worker_num=100, worker_cpu=4, worker_mem=16 * 1024 ** 3)
+    session = mars.new_ray_session(supervisor_cpu=16)