Merge branch 'master' into feature/mldataset

.github/workflows/cancel-prev.yml

@@ -9,4 +9,4 @@ jobs:
       - uses: styfle/cancel-workflow-action@0.8.0
         with:
           access_token: ${{ github.token }}
-          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml
+          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml,checks.yml

.github/workflows/checks.yml

@@ -0,0 +1,70 @@
+name: Pre-commit Checks
+
+on: [push, pull_request]
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Set up conda ${{ matrix.python-version }}
+        env:
+          PYTHON: ${{ matrix.python-version }}
+        shell: bash
+        run: |
+          source ./ci/install-conda.sh
+          python -m pip install --upgrade pip setuptools wheel coverage;
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          export DEFAULT_VENV=$VIRTUAL_ENV
+
+          pip install numpy scipy cython
+          pip install -e ".[dev,extra]"
+          pip install virtualenv flake8 codespell sphinx sphinx-intl
+          conda list -n test
+
+      - name: Lint with flake8
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 mars --count --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 mars --config="default" --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Check spellings
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          codespell
+
+      - name: Check copyright headers
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          python ci/copycheck.py
+
+      - name: Check JS with ESLint
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          pushd mars/services/web/ui
+          npm install
+          npm run lint
+          popd
+
+      - name: Check documentation build
+        shell: bash
+        run: |
+          source ./ci/reload-env.sh
+          pushd docs
+          pip install -r requirements-doc.txt
+          make html
+          popd

.github/workflows/core-ci.yml

@@ -12,7 +12,7 @@ jobs:
         python-version: [3.6, 3.7, 3.8, 3.9, 3.8-cython]
         include:
           - { os: ubuntu-latest, python-version: 3.8-cython, no-common-tests: 1,
-              no-deploy: 1, with-cython: 1, with-flake8: 1 }
+              no-deploy: 1, with-cython: 1 }
 
     steps:
     - name: Check out code
@@ -53,7 +53,7 @@ jobs:
         pip install git+https://github.com/mars-project/pytest-asyncio.git
         pip install numpy scipy cython
         pip install -e ".[dev,extra]"
-        pip install virtualenv flaky flake8
+        pip install virtualenv flaky
 
         if [ -z "$NO_COMMON_TESTS" ]; then
           if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
@@ -79,16 +79,6 @@ jobs:
         retry ./.github/workflows/download-etcd.sh
         conda list -n test
 
-    - name: Lint with flake8
-      shell: bash
-      if: success() && matrix.with-flake8 == 1
-      run: |
-        source ./ci/reload-env.sh
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 mars --count --select=E9,E111,E225,E302,E303,E901,E999,F7,F63,F82,F401,F811,F821,F822,F823,F841,W291,W292,W391,W605 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 mars --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
     - name: Test with pytest
       env:
         WITH_HADOOP: ${{ matrix.with-hadoop }}

.github/workflows/os-compat-ci.yml

@@ -40,7 +40,7 @@ jobs:
         pip install git+https://github.com/mars-project/pytest-asyncio.git
         pip install numpy scipy cython
         pip install -e ".[dev,extra]"
-        pip install virtualenv flaky flake8
+        pip install virtualenv flaky
         conda list -n test
 
     - name: Test with pytest

.github/workflows/platform-ci.yml

@@ -59,9 +59,9 @@ jobs:
           pip install -e ".[dev,extra]"
 
           if [[ $UNAME == "windows" ]]; then
-            pip install virtualenv flaky flake8
+            pip install virtualenv flaky
           else
-            pip install virtualenv flaky flake8 ray
+            pip install virtualenv flaky ray
             if [ -n "$WITH_KUBERNETES" ]; then
               ./.github/workflows/install-minikube.sh
               pip install kubernetes

.github/workflows/upload-packages.sh

@@ -40,7 +40,7 @@ else
     cp *.whl dist/
 
     if [[ "$UNAME" == "darwin" ]]; then
-      pip install delocate
+      pip install delocate==0.8.2
       delocate-wheel dist/*.whl
       delocate-addplat --rm-orig -x 10_9 -x 10_10 dist/*.whl
     fi

README.rst

@@ -30,7 +30,7 @@ for development:
     pip install -e ".[dev]"
 
 More details about installing Mars can be found at
-`getting started <https://docs.pymars.org/en/latest/installation/index.html>`_ section in
+`installation <https://docs.pymars.org/en/latest/installation/index.html>`_ section in
 Mars document.
 
 

azure-pipelines.yml

@@ -70,7 +70,7 @@ jobs:
           pip install numpy scipy cython
         fi
         pip install -e ".[dev,extra]"
-        pip install virtualenv flaky flake8
+        pip install virtualenv flaky
 
         if [ -z "$NO_COMMON_TESTS" ]; then
           if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then

ci/copycheck.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from pathlib import PurePath
+
+_MATCH_FILES = [
+    '*.py',
+    '*.pyx',
+]
+_IGNORES = [
+    'mars/learn/**/*.pyx',
+    'mars/lib/**/*.py',
+]
+
+
+def main():
+    root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    miss_files = []
+    for root, _dirs, files in os.walk(os.path.join(root_path, 'mars')):
+        for fn in files:
+            rel_path = os.path.relpath(os.path.join(root, fn), root_path)
+            if any(PurePath(rel_path).match(patt) for patt in _IGNORES):
+                continue
+            if all(not PurePath(rel_path).match(patt) for patt in _MATCH_FILES):
+                continue
+
+            file_path = os.path.join(root, fn)
+            with open(file_path, 'rb') as input_file:
+                file_lines = [line for line in input_file.read().split(b'\n')
+                              if line.startswith(b'#')]
+            comments = b'\n'.join(file_lines)
+            if b'Copyright' not in comments:
+                miss_files.append(rel_path)
+    if miss_files:
+        file_list = '\n    '.join(miss_files)
+        sys.stderr.write(f'Please add missing copyright header for files:\n'
+                         f'    {file_list}\n')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

docs/source/development/contributing.rst

@@ -74,6 +74,16 @@ the root of Mars project:
 
     python setup.py build_ext -i
 
+Rebuilding Frontend Code
+````````````````````````
+Mars uses `React <https://reactjs.org>`_ to build its frontend. You need to
+install `nodejs <https://nodejs.org>`_ to build it from source. After all
+dependencies installed, simply use command below to build your frontend code:
+
+.. code-block:: bash
+
+    python setup.py build_web
+
 Running Tests
 -------------
 It is recommended to use ``pytest`` to run Mars tests. A simple command below
@@ -102,6 +112,16 @@ before running the pytest command mentioned above. After report is generated,
 it it recommended to remove all generated C files and binaries and rebuild
 without ``CYTHON_TRACE``, as this option will reduce the performance of Mars.
 
+Check Code Styles
+-----------------
+Before proposing changes to Mars, you need to make sure your code style meets
+our requirements. Mars uses `flake8 <https://flake8.pycqa.org>`_ to check
+Python code style.  Simply run command below to check your code style:
+
+.. code-block:: bash
+
+    flake8 mars
+
 Building Documentations
 -----------------------
 Mars uses ``sphinx`` to build documents. You need to install necessary packages

docs/source/development/oscar/batch.rst

@@ -0,0 +1,62 @@
+.. _batch_index:
+
+Create and Use Batch Methods
+============================
+
+Oscar provides a set of APIs to write batch methods. You can simply add a
+``@extensible`` decorator to your actor method and create a batch version.  All
+calls wrapped in a batch will be sent together, reducing possible RPC cost.
+
+Create a Batch Method
+---------------------
+
+You can create a batch method with a ``@extensible`` decorator:
+
+.. code-block:: python
+
+    import mars.oscar as mo
+
+    class ExampleActor(mo.Actor):
+        @mo.extensible
+        async def batch_method(self, a, b=None):
+            pass
+
+Sometimes we need to process received batch requests. For instance, we need to
+group requests by certain keys and resent them to different handlers in
+batches. Oscar supports creating a batch version of the method:
+
+.. code-block:: python
+
+    class ExampleActor(mo.Actor):
+        @mo.extensible
+        async def batch_method(self, a, b=None):
+            raise NotImplementedError  # this will redirect all requests to the batch version
+
+        @batch_method.batch
+        async def batch_method(self, args_list, kwargs_list):
+            results = []
+            for args, kwargs in zip(args_list, kwargs_list):
+                a, b = self.batch_method.bind(*args, **kwargs)
+                # process the request
+                results.append(result)
+            return results  # return a list of results
+
+In the code above, we simply raises a ``NotImplementedError`` to let the batch
+version handle all requests. The batch version have two arguments accepting
+``args`` and ``kwargs`` of all batched calls as lists. To make argument
+extraction easier, a utility function ``bind`` is added as an attribute of the
+method which extracts ``args`` and ``kwargs`` into real arguments.
+
+Call Batch Methods
+------------------
+
+Calling batch methods is easy. You can use ``<method_name>.delay`` to make a
+batched call and use ``<method_name>.batch`` to send them:
+
+.. code-block:: python
+
+    ref = await mo.actor_ref(uid='ExampleActor', address='127.0.0.1:13425')
+    results = await ref.batch_method.batch(
+        ref.batch_method.delay(10, b=20),
+        ref.batch_method.delay(20),
+    )

docs/source/development/oscar/index.rst

@@ -10,4 +10,5 @@ for entire distributed layer.
    :maxdepth: 2
 
    usage
+   batch
    internals

docs/source/development/services/cluster.rst

@@ -3,6 +3,20 @@
 Cluster Service
 ===============
 
+Cluster service manages nodes in a Mars cluster and expose APIs for
+other services to watch changes inside the cluster. The service uses different cluster backends
+for different systems.
+
+Configuration
+-------------
+.. code-block:: yaml
+
+    cluster:
+        backend: "<cluster backend name>"
+        lookup_address: "<address of master>"
+        node_timeout: "timeout seconds of nodes"
+        node_check_interval: "check interval seconds for nodes"
+
 APIs
 ----
 
@@ -11,5 +25,6 @@ APIs
 .. autosummary::
    :toctree: generated/
 
+   AbstractClusterBackend
    ClusterAPI
    WebClusterAPI

docs/source/development/services/meta.rst

@@ -3,6 +3,15 @@
 Meta Service
 ============
 
+Configuration
+-------------
+
+.. code-block:: yaml
+
+    meta:
+        store: "<meta store name>"
+        # other config related to each store
+
 APIs
 ----
 

docs/source/development/services/scheduling.rst

@@ -43,6 +43,17 @@ operand graph.  We choose the worker of descendant operands given the location
 of input chunks. When there are multiple workers providing minimal network
 transfer, a worker satisfying resource requirements are selected.
 
+Configuration
+-------------
+
+.. code-block:: yaml
+
+    scheduling:
+        mem_quota_size: "80%",
+        mem_hard_limit: "95%",
+        enable_kill_slot: true,
+        subtask_max_retries": 1
+
 APIs
 ----
 

docs/source/development/services/storage.rst

@@ -3,6 +3,15 @@
 Storage Service
 ===============
 
+Configuration
+-------------
+
+.. code-block:: yaml
+
+    storage:
+        backends: ["plasma"]
+        "<storage backend name>"： "<setup params>"
+
 APIs
 ----