feat: add FsspecConnector to easily integrate new connectors with a fsspec implementation available

.github/workflows/ci.yml

@@ -5,68 +5,67 @@ on:
   # We can switch to running on push if we make this repo public or are fine with
   # paying for CI minutes.
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
-
+    branches: [main]
 
 jobs:
   setup:
     strategy:
       matrix:
-        python-version: ["3.8","3.9","3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
     runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
-      id: virtualenv-cache
-      with:
-        path: |
-          .venv
-          nltk_data
-        key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }}
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Setup virtual environment (no cache hit)
-      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
-      run: |
-        python${{ matrix.python-version }} -m venv .venv
-        source .venv/bin/activate
-        make install-ci
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        id: virtualenv-cache
+        with:
+          path: |
+            .venv
+            nltk_data
+          key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup virtual environment (no cache hit)
+        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
+        run: |
+          python${{ matrix.python-version }} -m venv .venv
+          source .venv/bin/activate
+          make install-ci
 
   lint:
     strategy:
       matrix:
-        python-version: ["3.8","3.9","3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
     runs-on: ubuntu-latest
     needs: setup
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
-      id: virtualenv-cache
-      with:
-        path: .venv
-        key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }}
-    # NOTE(robinson) - This is a fallback in case the lint job does not find the cache.
-    # We can take this out when we implement the fix in CORE-99
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Setup virtual environment (no cache hit)
-      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
-      run: |
-        python${{ matrix.python-version }} -m venv .venv
-        source .venv/bin/activate
-        make install-ci
-    - name: Lint
-      run: |
-        source .venv/bin/activate
-        make check
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        id: virtualenv-cache
+        with:
+          path: .venv
+          key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }}
+      # NOTE(robinson) - This is a fallback in case the lint job does not find the cache.
+      # We can take this out when we implement the fix in CORE-99
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup virtual environment (no cache hit)
+        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
+        run: |
+          python${{ matrix.python-version }} -m venv .venv
+          source .venv/bin/activate
+          make install-ci
+      - name: Lint
+        run: |
+          source .venv/bin/activate
+          make check
 
   shellcheck:
     runs-on: ubuntu-latest
@@ -78,50 +77,53 @@ jobs:
   test:
     strategy:
       matrix:
-        python-version: ["3.8","3.9","3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
     runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup, lint]
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
-      id: virtualenv-cache
-      with:
-        path: |
-          .venv
-          nltk_data
-        key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }}
-    # NOTE(robinson) - This is a fallback in case the lint job does not find the cache.
-    # We can take this out when we implement the fix in CORE-99
-    - name: Setup virtual environment (no cache hit)
-      if: steps.virtualenv-cache.outputs.cache-hit != 'true'
-      run: |
-        python${{ matrix.python-version}} -m venv .venv
-        source .venv/bin/activate
-        make install-ci
-    - name: Test
-      run: |
-        source .venv/bin/activate
-        make install-detectron2
-        sudo apt-get install -y libmagic-dev poppler-utils tesseract-ocr libreoffice
-        make test
-        make check-coverage
-        make install-ingest-s3
-        make install-ingest-github
-        make install-ingest-wikipedia
-        ./test_unstructured_ingest/test-ingest.sh
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        id: virtualenv-cache
+        with:
+          path: |
+            .venv
+            nltk_data
+          key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/*.txt') }}
+      # NOTE(robinson) - This is a fallback in case the lint job does not find the cache.
+      # We can take this out when we implement the fix in CORE-99
+      - name: Setup virtual environment (no cache hit)
+        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
+        run: |
+          python${{ matrix.python-version}} -m venv .venv
+          source .venv/bin/activate
+          make install-ci
+      # NOTE: we need to add `make install-ingest-...` for each connector, and some are currently missing
+      # because the tests have not been created e.g. gcs or azure (abs), once a public remote path is available
+      # for testing purposes, the tests can be developed and triggered as part of the CI.
+      - name: Test
+        run: |
+          source .venv/bin/activate
+          make install-detectron2
+          sudo apt-get install -y libmagic-dev poppler-utils tesseract-ocr libreoffice
+          make test
+          make check-coverage
+          make install-ingest-s3
+          make install-ingest-github
+          make install-ingest-wikipedia
+          ./test_unstructured_ingest/test-ingest.sh
 
   changelog:
     runs-on: ubuntu-latest
     steps:
-    - if: github.ref != 'refs/heads/main'
-      uses: dorny/paths-filter@v2
-      id: changes
-      with:
-        filters: |
-          src:
-            - 'unstructured/**'
+      - if: github.ref != 'refs/heads/main'
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'unstructured/**'
 
-    - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
-      uses: dangoslen/changelog-enforcer@v3
+      - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main'
+        uses: dangoslen/changelog-enforcer@v3

.github/workflows/codeql-analysis.yml

@@ -13,12 +13,12 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "main" ]
+    branches: ["main"]
   schedule:
-    - cron: '21 21 * * 5'
+    - cron: "21 21 * * 5"
 
 jobs:
   analyze:
@@ -32,43 +32,42 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'python' ]
+        language: ["python"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
 
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
-      with:
-        category: "/language:${{matrix.language}}"
+      # - run: |
+      #   echo "Run, Build Application using script"
+      #   ./location_of_script_within_repo/buildscript.sh
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/sphinx.yml

@@ -5,23 +5,23 @@ name: Sphinx build
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
 
 jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: Build HTML
-      uses: ammaraskar/sphinx-action@e781e9af3e80bfe0ea539e4ea46858d51e027214
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: html-docs
-        path: docs/build/html/
-    - name: Deploy
-      uses: peaceiris/actions-gh-pages@v3
-      if: github.ref == 'refs/heads/main'
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: docs/build/html
+      - uses: actions/checkout@v3
+      - name: Build HTML
+        uses: ammaraskar/sphinx-action@e781e9af3e80bfe0ea539e4ea46858d51e027214
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-docs
+          path: docs/build/html/
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.ref == 'refs/heads/main'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/build/html

.gitignore

@@ -182,3 +182,6 @@ tags
 [._]*.un~
 
 .DS_Store
+
+# Ruff cache
+.ruff_cache/

CHANGELOG.md

@@ -1,9 +1,20 @@
-## 0.5.2-dev1
+## 0.5.2-dev2
 
 ### Enhancements
 
+* Add `FsspecConnector` to easily integrate any existing `fsspec` filesystem
+as a connector.
+* Rename `s3_connector.py` to `s3.py` for readability and consistency with the
+rest of the connectors.
+
 ### Features
 
+* Update `S3Connector` to inherit from `FsspecConnector`
+* Add `GCSConnector`  (missing CLI integration and working example but has been tested
+with a private bucket with a PDF and works as expected)
+* Add `ABSConnector` (missing CLI integration and working example but has been tested
+with a private container with a PDF and works as expected)
+
 ### Fixes
 
 ## 0.5.1

Ingest.md

@@ -67,7 +67,7 @@ In checklist form, the above steps are summarized as:
   - [ ] Add them as an extra to [setup.py](unstructured/setup.py).
   - [ ] Update the Makefile, adding a target for `install-ingest-<name>` and adding another `pip-compile` line to the `pip-compile` make target. See [this commit](https://github.com/Unstructured-IO/unstructured/commit/ab542ca3c6274f96b431142262d47d727f309e37) for a reference.
   - [ ] The added dependencies should be imported at runtime when the new connector is invoked, rather than as top-level imports.
-  - [ ] Add the decorator `unstructured.utils.requires_dependencies` on top of each class instance or function that uses those connector-specific dependencies e.g. for `S3Connector` should look like `@requires_dependencies(dependencies=["boto3"], extras="s3")`
+  - [ ] Add the decorator `unstructured.utils.requires_dependencies` on top of each class instance or function that uses those connector-specific dependencies e.g. for `S3Connector` should look like `@requires_dependencies(dependencies=["s3fs", "fsspec"], extras="s3")`
 - [ ] Honors the conventions of `BaseConnectorConfig` defined in [unstructured/ingest/interfaces.py](unstructured/ingest/interfaces.py) which is passed through [the CLI](unstructured/ingest/main.py):
   - [ ] If running with an `.output_dir` where structured outputs already exists for a given file, the file content is not re-downloaded from the data source nor is it reprocessed. This is made possible by implementing the call to `MyIngestDoc.has_output()` which is invoked in [MainProcess._filter_docs_with_outputs](ingest-prep-for-many/unstructured/ingest/main.py).
   - [ ] Unless `.reprocess` is `True`, then documents are always reprocessed.

Makefile

@@ -49,11 +49,18 @@ install-dev:
 install-build:
 	pip install -r requirements/build.txt
 
-## install-ingest-s3:       install requirements for the s3 connector
 .PHONY: install-ingest-s3
 install-ingest-s3:
 	pip install -r requirements/ingest-s3.txt
 
+.PHONY: install-ingest-gcs
+install-ingest-gcs:
+	pip install -r requirements/ingest-gcs.txt
+
+.PHONY: install-ingest-azure
+install-ingest-azure:
+	pip install -r requirements/ingest-azure.txt
+
 .PHONY: install-ingest-github
 install-ingest-github:
 	pip install -r requirements/ingest-github.txt
@@ -95,6 +102,8 @@ pip-compile:
 	# sphinx docs looks for additional requirements
 	cp requirements/build.txt docs/requirements.txt
 	pip-compile --upgrade --extra=s3        --output-file=requirements/ingest-s3.txt        requirements/base.txt setup.py
+	pip-compile --upgrade --extra=gcs       --output-file=requirements/ingest-gcs.txt       requirements/base.txt setup.py
+	pip-compile --upgrade --extra=azure     --output-file=requirements/ingest-azure.txt     requirements/base.txt setup.py
 	pip-compile --upgrade --extra=reddit    --output-file=requirements/ingest-reddit.txt    requirements/base.txt setup.py
 	pip-compile --upgrade --extra=github    --output-file=requirements/ingest-github.txt    requirements/base.txt setup.py
 	pip-compile --upgrade --extra=wikipedia --output-file=requirements/ingest-wikipedia.txt requirements/base.txt setup.py