Build pylance on Mac and Update python build instruction. (milvus-io#19)

.gitignore

@@ -36,7 +36,11 @@
 **/__pycache__
 build/
 dist/
+*.egg-info/
 
 .idea
 cmake-build-*
 .vscode
+
+python/lance/_lib.cpp
+

python/MANIFEST.in

@@ -0,0 +1 @@
+include README.md

python/README.md

@@ -0,0 +1,64 @@
+# Setup
+
+# Env
+- DON'T use conda as it prefers it's on ld path and libstd etc
+- Remaining instructions are for Ubuntu only
+
+```bash
+sudo apt install python3-pip python3-venv python3-dev
+python3 -m venv ${HOME}/.venv/nft
+```
+
+# Arrow C++ libs
+
+As a shortcut we won't build the arrow c++ libs from scratch.
+Instead, follow the [arrow installation instructions](https://arrow.apache.org/install/).
+These instructions don't include `libarrow-python-dev` so that needs to be apt installed
+separately.
+
+# Build pyarrow
+
+Assume CWD is where you want to put the repo:
+
+```bash
+source ${HOME}/.venv/nft/bin/activate
+cd /path/to/lance/python/thirdparty
+./build.sh
+```
+
+Make sure pyarrow works properly:
+
+```python
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pyarrow.dataset as ds
+```
+
+# Build Lance
+
+Assume CWD is where you want to put the repo:
+
+```bash
+git clone git@github.com:eto-ai/lance
+
+pushd nft/cpp
+cmake . -B build
+pushd build
+make -j
+popd
+popd
+
+pushd python
+source ${HOME}/.venv/nft/bin/activate
+python setup.py develop
+```
+
+Test the installation in python:
+
+```python
+import duckdb
+import pylance
+uri = "..../pet.lance"
+pets = pylance.dataset(uri)
+duckdb.query('select label, count(1) from pets group by label').to_arrow_table()
+```

python/pylance/__init__.py → python/lance/__init__.py

@@ -1,5 +1,5 @@
 import pyarrow.dataset as ds
-from pylance.lib import LanceFileFormat
+from lance.lib import LanceFileFormat
 
 
 def dataset(uri: str):

python/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel",
+    "ninja"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.isort]
+profile = "black"
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+filterwarnings = ["error"]
+testpaths = ["pylance/tests"]
+
+[tool.cibuildwheel]
+test-command = "pytest {project}/pylance/tests"
+test-extras = ["test"]
+test-skip = ["*universal2:arm64"]
+# Setuptools bug causes collision between pypy and cpython artifacts
+before-build = "rm -rf {project}/build"

python/setup.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+from setuptools import Extension, find_packages, setup
+
+import numpy as np
+import pyarrow as pa
+from Cython.Build import cythonize
+
+arrow_includes = pa.get_include()
+numpy_includes = np.get_include()
+
+# TODO allow for custom liblance directory
+lance_cpp = Path(__file__).resolve().parent.parent / 'cpp'
+lance_includes = str(lance_cpp / 'include')
+lance_libs = str(lance_cpp / 'build')
+
+
+extensions = [Extension(
+    "lance.lib",
+    ["lance/_lib.pyx"],
+    include_dirs=[lance_includes, arrow_includes, numpy_includes],
+    libraries=['lance'],
+    library_dirs=[lance_libs],
+    language="c++",
+    extra_compile_args=["-Wall", "-std=c++20", "-O3"],
+    extra_link_args=["-Wl,-rpath", lance_libs]
+)]
+
+
+# The information here can also be placed in setup.cfg - better separation of
+# logic and declaration, and simpler if you include description/version in a file.
+setup(
+    name="pylance",
+    version="0.0.1",
+    author="Lance Developers",
+    author_email="contact@eto.ai",
+    description="Python extension for lance",
+    long_description="",
+    ext_modules=cythonize(extensions, language_level="3"),
+    zip_safe=False,
+    extras_require={"test": ["pytest>=6.0"]},
+    python_requires=">=3.8",
+    packages=find_packages()
+)

python/thirdparty/build.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -ex
+
+APACHE_ARROW_VERSION=release-8.0.0
+
+# Build apache arrow
+function build_arrow {
+    git clone git@github.com:apache/arrow
+    pushd arrow
+    git checkout ${APACHE_ARROW_VERSION}
+    git submodule update --init
+    pushd python
+    pip install -r requirements-build.txt
+
+    export OPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl@1.1
+    python setup.py build_ext --inplace --with-dataset --with-parquet --with-s3
+    python setup.py develop
+}
+
+build_arrow