Merge branch 'master' into add-arrow-datasource-and-project-docs

Author: allisonwang-db

.github/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+    
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        
+    - name: Install Poetry
+      uses: snok/install-poetry@v1.3.4
+      with:
+        version: latest
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+        
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction --no-root --extras "all"
+      
+    - name: Install project
+      run: poetry install --no-interaction --extras "all"
+      
+    - name: Run tests
+      run: poetry run pytest tests/ -v
+      
+    - name: Run tests with coverage
+      run: |
+        poetry run pytest tests/ --cov=pyspark_datasources --cov-report=xml --cov-report=term-missing
+  

.gitignore

@@ -159,5 +159,9 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 
-# Claude
+# Claude Code
 .claude/
+claude_cache/
+
+# Gemini
+.gemini/

README.md

@@ -46,6 +46,7 @@ spark.readStream.format("fake").load().writeStream.format("console").start()
 | [KaggleDataSource](pyspark_datasources/kaggle.py)                      | `kaggle`       | Read datasets from Kaggle                    | `kagglehub`, `pandas` |
 | [SimpleJsonDataSource](pyspark_datasources/simplejson.py)              | `simplejson`   | Write JSON data to Databricks DBFS                 | `databricks-sdk`      |
 | [OpenSkyDataSource](pyspark_datasources/opensky.py)                 | `opensky`      | Read from OpenSky Network.                   | None                  |
+| [SalesforceDataSource](pyspark_datasources/salesforce.py)              | `salesforce`   | Streaming sink for writing data to Salesforce | `simple-salesforce`   |
 
 See more here: https://allisonwang-db.github.io/pyspark-data-sources/.
 

docs/datasources/salesforce.md

@@ -0,0 +1,6 @@
+# SalesforceDataSource
+
+> Requires the [`simple-salesforce`](https://github.com/simple-salesforce/simple-salesforce) library. You can install it manually: `pip install simple-salesforce`
+> or use `pip install pyspark-data-sources[salesforce]`.
+
+::: pyspark_datasources.salesforce.SalesforceDataSource

docs/index.md

@@ -38,5 +38,6 @@ spark.readStream.format("fake").load().writeStream.format("console").start()
 | [HuggingFaceDatasets](./datasources/huggingface.md)     | `huggingface`  | Read datasets from the HuggingFace Hub        | `datasets`            |
 | [StockDataSource](./datasources/stock.md)               | `stock`        | Read stock data from Alpha Vantage            | None                  |
 | [SimpleJsonDataSource](./datasources/simplejson.md)     | `simplejson`   | Write JSON data to Databricks DBFS            | `databricks-sdk`      |
+| [SalesforceDataSource](./datasources/salesforce.md)     | `salesforce`   | Write streaming data to Salesforce objects    |`simple-salesforce`    |
 | [GoogleSheetsDataSource](./datasources/googlesheets.md) | `googlesheets` | Read table from public Google Sheets document | None                  |
 | [KaggleDataSource](./datasources/kaggle.md)             | `kaggle`       | Read datasets from Kaggle                     | `kagglehub`, `pandas` |