[Feat][Spark] Implementation of PySpark bindings to Scala API

.github/workflows/pyspark.yml

@@ -0,0 +1,62 @@
+# Copyright 2022-2023 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: GraphAr PySpark CI
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  GraphAr-spark:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          submodules: true
+
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
+    - name: Install Poetry
+      uses: abatilo/actions-poetry@v2
+
+    - name: Install Spark Scala && PySpark
+      run: |
+        cd pyspark
+        make install_test
+
+    - name: Run PyTest
+      run: |
+        cd pyspark
+        make test
+
+    - name: Lint
+      run: |
+        cd pyspark
+        make install_lint
+        make lint
+

.gitignore

@@ -6,3 +6,66 @@
 .ccls-cache
 
 compile_commands.json
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+pyspark/assets
+
+# Jupyter Notebook
+.ipynb_checkpoints
+*.ipynb
+
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Ruff
+.ruff_cache
+
+### Scala ###
+*.bloop
+*.metals

.licenserc.yaml

@@ -40,11 +40,12 @@ header:
     - '*.md'
     - '*.rst'
     - '**/*.json'
+    - 'pyspark/poetry.lock' # This file is generated automatically by Poetry-tool; there is no way to add license header
 
   comment: on-failure
 
 # If you don't want to check dependencies' license compatibility, remove the following part
 dependency:
   files:
     - spark/pom.xml           # If this is a maven project.
-    - java/pom.xml           # If this is a maven project.
+    - java/pom.xml           # If this is a maven project.

docs/Makefile

@@ -53,3 +53,25 @@ html: cpp-apidoc spark-apidoc
 				--quiet
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: pyspark-apidoc
+pyspark-apidoc:
+	cd $(ROOTDIR)/pyspark && \
+	poetry run sphinx-apidoc -o $(ROOTDIR)/docs/pyspark/api graphar_pyspark/
+
+.PHONY: html-poetry
+html-poetry:
+	cd $(ROOTDIR)/pyspark && \
+	poetry run bash -c "cd $(ROOTDIR)/docs && $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html"
+	rm -fr $(BUILDDIR)/html/spark/reference
+	cp -fr $(ROOTDIR)/spark/target/site/scaladocs $(BUILDDIR)/html/spark/reference/
+	cd $(ROOTDIR)/java && \
+	mvn -P javadoc javadoc:aggregate \
+				-Dmaven.antrun.skip=true \
+				-DskipTests \
+				-Djavadoc.output.directory=$(ROOTDIR)/docs/$(BUILDDIR)/html/java/ \
+				-Djavadoc.output.destDir=reference \
+				--quiet
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+

docs/index.rst

@@ -21,6 +21,7 @@
    C++ <cpp/index>
    Java <java/index>
    Spark <spark/index>
+   PySpark <pyspark/index>
 
 .. toctree::
    :maxdepth: 2

docs/pyspark/api/graphar_pyspark.rst

@@ -0,0 +1,69 @@
+graphar\_pyspark package
+========================
+
+Submodules
+----------
+
+graphar\_pyspark.enums module
+-----------------------------
+
+.. automodule:: graphar_pyspark.enums
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+graphar\_pyspark.errors module
+------------------------------
+
+.. automodule:: graphar_pyspark.errors
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+graphar\_pyspark.graph module
+-----------------------------
+
+.. automodule:: graphar_pyspark.graph
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+graphar\_pyspark.info module
+----------------------------
+
+.. automodule:: graphar_pyspark.info
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+graphar\_pyspark.reader module
+------------------------------
+
+.. automodule:: graphar_pyspark.reader
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+graphar\_pyspark.util module
+----------------------------
+
+.. automodule:: graphar_pyspark.util
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+graphar\_pyspark.writer module
+------------------------------
+
+.. automodule:: graphar_pyspark.writer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: graphar_pyspark
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/pyspark/api/modules.rst

@@ -0,0 +1,7 @@
+graphar_pyspark
+===============
+
+.. toctree::
+   :maxdepth: 4
+
+   graphar_pyspark