REF: Move mars into xorbits (#195)

.dockerignore

@@ -4,5 +4,4 @@
 */.mypy_cache
 */.pytest_cache
 */build
-*/*/_mars
 **/node_modules

.github/workflows/python.yaml

@@ -43,9 +43,11 @@ jobs:
       - uses: isort/isort-action@master
         with:
           sortPaths: "python/xorbits"
-          configuration: "--check-only --profile black --diff --skip-glob xorbits/_mars/"
+          configuration: "--check-only --profile black --diff --skip python/xorbits/_mars/"
       - name: mypy
         run: pip install mypy && cd python && mypy xorbits
+      - name: codespell
+        run: pip install codespell && cd python && codespell xorbits
       - name: Set up Node.js
         uses: actions/setup-node@v1
         with:

.gitignore

@@ -119,6 +119,16 @@ venv.bak/
 # mkdocs documentation
 /site
 
+# cython compiled files
+python/xorbits/_mars/*.c*
+python/xorbits/_mars/*.h*
+python/xorbits/_mars/core/**/*.c*
+python/xorbits/_mars/learn/cluster/*.c*
+python/xorbits/_mars/learn/utils/*.c*
+python/xorbits/_mars/lib/*.c*
+python/xorbits/_mars/oscar/**/*.c*
+python/xorbits/_mars/serialization/*.c*
+
 # mypy
 .mypy_cache/
 .dmypy.json
@@ -132,9 +142,6 @@ dmypy.json
 .vscode
 *.iml
 
-# soft link
-python/xorbits/_mars
-
 # web staff
 node_modules/
 static/

.pre-commit-config.yaml

@@ -18,15 +18,23 @@ repos:
     rev: 5.12.0
     hooks:
       - id: isort
-        args: [--profile=black]
+        args: [--sp, python/setup.cfg]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.0.0
     hooks:
       - id: mypy
         additional_dependencies: [tokenize-rt==3.2.0]
-        args: [--config-file, python/setup.cfg]
+        exclude: _mars
+        args: [--ignore-missing-imports, --follow-imports, skip]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.0-alpha.4 # Use the sha or tag you want to point at
     hooks:
       - id: prettier
         types_or: [html, javascript]
+        args: [--ignore-path, python/xorbits/_mars]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.2
+    hooks:
+      - id: codespell
+        exclude: _mars/lib
+        args: [ --config, python/setup.cfg]

python/setup.cfg

@@ -168,13 +168,16 @@ exclude =
     ci/
     dist/
     docs/
-    xorbits/_mars/*
+    xorbits/_mars/lib/nvutils.py
+    xorbits/_mars/lib/uhashring/*
+    xorbits/_mars/lib/version.py
+
 per-file-ignores =
      */core/adapter.py: F401
 
 [codespell]
 ignore-words-list = hist,rcall,fpr,ser,nd,inout,ot,Ba,ba,asend,hart,coo,splitted,datas,fro
-skip = .idea,.git,./build,./docs/build,./xorbits/_mars/lib,node_modules,static,generated,*.po,*.ts,*.json,*.c,*.cpp,*.cfg
+skip = .idea,.git,./build,./docs/build,xorbits/_mars/lib,node_modules,static,generated,*.po,*.ts,*.json,*.c,*.cpp,*.cfg
 
 [isort]
 profile = black

python/setup.py

@@ -61,15 +61,6 @@
 repo_root = os.path.dirname(os.path.abspath(__file__))
 os.chdir(repo_root)
 
-# create symlink for mars
-absolute_path = os.path.join(repo_root, os.path.join("xorbits", "_mars"))
-source_path = os.path.join("..", "..", "third_party", "_mars", "mars")
-try:
-    os.symlink(source_path, absolute_path, target_is_directory=True)
-except FileExistsError:
-    # symlink exists already, skip
-    pass
-
 
 cythonize_kw = dict(language_level=sys.version_info[0])
 cy_extension_kw = dict()

python/xorbits/_mars/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import _version
+from .config import options
+from .core.context import get_context
+from .deploy.oscar import new_cluster_in_ray, new_ray_session
+from .session import execute, fetch, fetch_log, new_session, stop_server
+
+__version__ = _version.get_versions()["version"]

python/xorbits/_mars/_resource.pyx

@@ -0,0 +1,73 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cdef class Resource:
+    cdef readonly:
+        float num_cpus
+        float num_gpus
+        float mem_bytes
+
+    def __init__(self, float num_cpus=0, float num_gpus=0, float mem_bytes=0):
+        self.num_cpus = num_cpus
+        self.num_gpus = num_gpus
+        self.mem_bytes = mem_bytes
+
+    def __eq__(self, Resource other):
+        cdef bint ret = (
+            self.mem_bytes == other.mem_bytes
+            and self.num_gpus == other.num_gpus
+            and self.num_cpus == other.num_cpus
+        )
+        return ret
+
+    cdef bint _le(self, Resource other) nogil:
+        # memory first, then gpu, cpu last
+        cdef bint ret = (
+            self.mem_bytes <= other.mem_bytes
+            and self.num_gpus <= other.num_gpus
+            and self.num_cpus <= other.num_cpus
+        )
+        return ret
+
+    def __gt__(self, Resource other):
+        return not self._le(other)
+
+    def __le__(self, Resource other):
+        return self._le(other)
+
+    def __add__(self, Resource other):
+        return Resource(
+            num_cpus=self.num_cpus + other.num_cpus,
+            num_gpus=self.num_gpus + other.num_gpus,
+            mem_bytes=self.mem_bytes + other.mem_bytes,
+        )
+
+    def __sub__(self, Resource other):
+        return Resource(
+            num_cpus=self.num_cpus - other.num_cpus,
+            num_gpus=self.num_gpus - other.num_gpus,
+            mem_bytes=self.mem_bytes - other.mem_bytes,
+        )
+
+    def __neg__(self):
+        return Resource(
+            num_cpus=-self.num_cpus,
+            num_gpus=-self.num_gpus,
+            mem_bytes=-self.mem_bytes,
+        )
+
+    def __repr__(self):
+        return f"Resource(num_cpus={self.num_cpus}, num_gpus={self.num_gpus}, mem_bytes={self.mem_bytes})"
+
+ZeroResource = Resource(num_cpus=0, num_gpus=0, mem_bytes=0)

python/xorbits/_mars/_utils.pxd

@@ -0,0 +1,33 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cdef class TypeDispatcher:
+    cdef dict _handlers
+    cdef dict _lazy_handlers
+    cdef dict _inherit_handlers
+    cdef object __weakref__
+
+    cpdef void register(self, object type_, object handler)
+    cpdef void unregister(self, object type_)
+    cdef _reload_lazy_handlers(self)
+    cpdef get_handler(self, object type_)
+
+
+cpdef str to_str(s, encoding=*)
+cpdef bytes to_binary(s, encoding=*)
+cpdef unicode to_text(s, encoding=*)
+cpdef register_tokenizer(cls, handler)
+cpdef void reset_id_random_seed() except *
+cpdef bytes new_random_id(int byte_len)