INTPYTHON-598 - Create an AI/ML helper library

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      actions:
+        patterns:
+          - "*"
+  # Python
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/dist.yml

@@ -0,0 +1,78 @@
+name: Python Dist
+
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
+  workflow_dispatch:
+  pull_request:
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: dist-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -eux {0}
+
+jobs:
+  make_dist:
+    name: Make Dist
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@v6
+        with:
+          # Build sdist on lowest supported Python
+          python-version: '3.10'
+
+      - name: Install python requirements
+        run: |
+          python -m pip install uv rust-just build
+
+      - name: Build Dist
+        run: |
+          python -m build .
+
+      - name: Test SDist
+        run: |
+          python -m pip install dist/*.gz
+          cd ..
+          python -c "from pymongo_vectorsearch_utils import create_vector_search_index"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "dist"
+          path: ./dist/*.*
+
+  collect_dist:
+    runs-on: ubuntu-latest
+    needs: [make_dist]
+    name: Download Dist
+    steps:
+      - name: Download all workflow run artifacts
+        uses: actions/download-artifact@v5
+      - name: Flatten directory
+        working-directory: .
+        run: |
+          find . -mindepth 2 -type f -exec mv {} . \;
+          find . -type d -empty -delete
+      - uses: actions/upload-artifact@v4
+        with:
+          name: all-dist-${{ github.run_id }}
+          path: "./*"

.github/workflows/release-python.yml

@@ -0,0 +1,127 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      following_version:
+        description: "The post (dev) version to set"
+      dry_run:
+        description: "Dry Run?"
+        default: false
+        type: boolean
+  schedule:
+    - cron:  '30 5 * * *'
+
+env:
+  # Changes per repo
+  PRODUCT_NAME: pymongo-vectorsearch-utils
+  # Constant
+  # inputs will be empty on a scheduled run.  so, we only set dry_run
+  # to 'false' when the input is set to 'false'.
+  DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }}
+  FOLLOWING_VERSION: ${{ inputs.following_version || '' }}
+
+concurrency:
+  group: wheels-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -eux {0}
+
+jobs:
+  pre-publish:
+    environment: release
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'mongodb-labs' || github.event_name == 'workflow_dispatch'
+    permissions:
+      id-token: write
+      contents: write
+    outputs:
+      version: ${{ steps.pre-publish.outputs.version }}
+    steps:
+      - uses: mongodb-labs/drivers-github-tools/secure-checkout@v2
+        with:
+          app_id: ${{ vars.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+      - uses: mongodb-labs/drivers-github-tools/setup@v2
+        with:
+          aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
+          aws_region_name: ${{ vars.AWS_REGION_NAME }}
+          aws_secret_id: ${{ secrets.AWS_SECRET_ID }}
+          artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }}
+      - uses: mongodb-labs/drivers-github-tools/python-labs/pre-publish@v2
+        id: pre-publish
+        with:
+          dry_run: ${{ env.DRY_RUN }}
+
+  build-dist:
+    needs: [pre-publish]
+    uses: ./.github/workflows/dist.yml
+    permissions:
+      contents: read
+      actions: read
+    with:
+      ref: ${{ needs.pre-publish.outputs.version }}
+
+  static-scan:
+    needs: [pre-publish]
+    uses: ./.github/workflows/codeql.yml
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+    with:
+      ref: ${{ needs.pre-publish.outputs.version }}
+
+  publish:
+    # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#publishing-the-distribution-to-pypi
+    needs: [build-dist, static-scan]
+    if: (github.repository_owner == 'mongodb-labs' && github.event_name != 'pull_request')  || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v5
+      with:
+        name: all-dist-${{ github.run_id }}
+        path: dist/
+    - name: Publish package distributions to TestPyPI
+      uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+        skip-existing: true
+        attestations: ${{ env.DRY_RUN }}
+    - name: Publish distribution 📦 to PyPI
+      if: startsWith(env.DRY_RUN, 'false')
+      uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1
+
+  post-publish:
+    needs: [publish]
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+      contents: write
+      attestations: write
+      security-events: write
+    steps:
+      - uses: mongodb-labs/drivers-github-tools/secure-checkout@v2
+        with:
+          app_id: ${{ vars.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+      - uses: mongodb-labs/drivers-github-tools/setup@v2
+        with:
+          aws_role_arn: ${{ secrets.AWS_ROLE_ARN }}
+          aws_region_name: ${{ vars.AWS_REGION_NAME }}
+          aws_secret_id: ${{ secrets.AWS_SECRET_ID }}
+          artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }}
+      - uses: mongodb-labs/drivers-github-tools/python-labs/post-publish@v2
+        with:
+          following_version: ${{ env.FOLLOWING_VERSION }}
+          product_name: ${{ env.PRODUCT_NAME }}
+          token: ${{ github.token }}
+          dry_run: ${{ env.DRY_RUN }}

.github/workflows/test-python.yml

@@ -0,0 +1,92 @@
+name: Python Tests
+
+on:
+  push:
+      branches: ["main"]
+  pull_request:
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash -eux {0}
+
+env:
+  MIN_PYTHON: "3.10"
+
+jobs:
+  static:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        persist-credentials: false
+        fetch-depth: 0
+    - name: Install uv
+      uses: astral-sh/setup-uv@557e51de59eb14aaaba2ed9621916900a91d50c6 # v5
+      with:
+        enable-cache: true
+        python-version: ${{ matrix.python-version }}
+    - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
+    - run: just install
+    - run: just lint
+    - run: just docs
+    - run: just doctest
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+      fail-fast: false
+    name: CPython ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@557e51de59eb14aaaba2ed9621916900a91d50c6 # v5
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+      - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
+      - uses: mongodb-labs/drivers-evergreen-tools@master
+        with:
+          local-atlas: '1'
+      - run: just install
+      - run: just test
+
+  build-min:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        persist-credentials: false
+        fetch-depth: 0
+    - name: Install uv
+      uses: astral-sh/setup-uv@557e51de59eb14aaaba2ed9621916900a91d50c6 # v5
+      with:
+        enable-cache: true
+        python-version: ${{ env.MIN_PYTHON }}
+    - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
+    - name: Install uv
+      uses: astral-sh/setup-uv@557e51de59eb14aaaba2ed9621916900a91d50c6 # v5
+      with:
+        enable-cache: true
+        python-version: ${{ env.MIN_PYTHON }}
+    - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3
+    - uses: mongodb-labs/drivers-evergreen-tools@master
+      with:
+        local-atlas: '1'
+    - name: Run unit tests with minimum dependency versions
+      run: |
+        uv sync --python=${MIN_PYTHON} --resolution=lowest-direct
+        just test

.github/workflows/zizmor.yml

@@ -0,0 +1,21 @@
+name: GitHub Actions Security Analysis with zizmor
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["**"]
+
+jobs:
+  zizmor:
+    name: zizmor latest via Cargo
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - name: Run zizmor
+        uses: zizmorcore/zizmor-action@a016d81e77496751b5c04eb1e8f00214bd396553

.github/zizmor.yml

@@ -0,0 +1,7 @@
+rules:
+  unpinned-uses:
+    config:
+      policies:
+        actions/*: ref-pin
+        mongodb-labs/drivers-github-tools/*: ref-pin
+        mongodb-labs/drivers-evergreen-tools: ref-pin

pymongo_vectorsearch_utils/__init__.py

@@ -1,6 +1,27 @@
 from ._version import __version__
-
+from .index import (
+    create_fulltext_search_index,
+    create_vector_search_index,
+    drop_vector_search_index,
+    update_vector_search_index,
+)
+from .pipeline import (
+    combine_pipelines,
+    final_hybrid_stage,
+    reciprocal_rank_stage,
+    text_search_stage,
+    vector_search_stage,
+)
 
 __all__ = [
     "__version__",
+    "create_vector_search_index",
+    "drop_vector_search_index",
+    "update_vector_search_index",
+    "create_fulltext_search_index",
+    "text_search_stage",
+    "vector_search_stage",
+    "combine_pipelines",
+    "reciprocal_rank_stage",
+    "final_hybrid_stage",
 ]