0.7.2 Release Candidate (#1297)

ArchiveBox · Jan 5, 2024 · 315c9f3 · 315c9f3
2 parents 1f33fc7 + 2ef3655
commit 315c9f3
Show file tree

Hide file tree

Showing 51 changed files with 2,123 additions and 877 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -5,17 +5,20 @@ __pycache__/
 .mypy_cache/
 .pytest_cache/
 .github/
-.git/
 .pdm-build/
 .pdm-python
 .eggs/
+.git/
+!.git/HEAD
+!.git/refs/heads/*
 
 venv/
 .venv/
 .venv-old/
 .docker-venv/
 node_modules/
 
+docs/
 build/
 dist/
 brew_dist/

diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml
@@ -25,36 +25,23 @@ jobs:
             dh-python debhelper devscripts dput software-properties-common \
             python3-distutils python3-setuptools python3-wheel python3-stdeb
 
-      - name: Build Debian/Apt sdist_dsc
-        run: |
-          rm -Rf deb_dist/*
-          python3 setup.py --command-packages=stdeb.command sdist_dsc
-
-      - name: Build Debian/Apt bdist_deb
-        run: |
-          python3 setup.py --command-packages=stdeb.command bdist_deb
-
-      - name: Install archivebox from deb
-        run: |
-          cd deb_dist/
-          sudo apt-get install ./archivebox*.deb
-          cd ..
-          python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > ./requirements.txt
-          python3 -m pip install -r ./requirements.txt
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
 
-      - name: Check ArchiveBox version
-        run: |
-          # must create dir needed for snaps to run as non-root on github actions
-          sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-          mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-          archivebox --version
-          archivebox init --setup
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
 
-      - name: Add some links to test
-        run: |
-          cd "${{ github.workspace }}/data"
-          archivebox add 'https://example.com'
-          archivebox status
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
 
       # - name: Commit built package
       #   run: |

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -11,8 +11,7 @@ on:
 
 env:
   DOCKER_IMAGE: archivebox-ci
-
-
+
 jobs:
   buildx:
     runs-on: ubuntu-latest
@@ -60,13 +59,12 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: archivebox/archivebox,nikisweeting/archivebox
-          flavor: |
-              latest=auto
           tags: |
               type=ref,event=branch
               type=semver,pattern={{version}}
               type=semver,pattern={{major}}.{{minor}}
               type=sha
+              type=raw,value=latest,enable={{is_default_branch}}
       
       - name: Build and push
         id: docker_build
@@ -78,8 +76,18 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
           platforms: linux/amd64,linux/arm64,linux/arm/v7
 
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
+
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml
@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml
@@ -19,19 +19,26 @@ jobs:
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: '3.11'
           architecture: x64
 
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
       - uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
       - name: Install dependencies
         run: pdm install --fail-fast --no-lock --group :all --no-self
 
       - name: Build package
         run: |
-          rm ./dist/archivebox-*.whl
           pdm build
 
       - name: Install from build
@@ -45,8 +52,8 @@ jobs:
           archivebox version
           archivebox status
 
-      - name: Publish package distributions to PyPI
-        run: pdm publish --no-build
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
 
       # - name: Push build to PyPI
       #   run: |

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -13,11 +13,12 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest, windows-latest]
-        python: [3.9]
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
@@ -29,13 +30,16 @@ jobs:
           python-version: ${{ matrix.python }}
           architecture: x64
 
-      - name: Set up Node JS 14.7.0
-        uses: actions/setup-node@v3
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          node-version: 18.12.0
+          node-version: 20.10.0
 
       - name: Setup PDM
         uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
       ### Install Python & JS Dependencies
       - name: Get pip cache dir
@@ -44,27 +48,32 @@ jobs:
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-pip
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
           key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
       - name: Install pip dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel pytest bottle build
-          ./bin/build_pip.sh
-          pdm install
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
 
       - name: Get npm cache dir
         id: npm-cache
         run: |
           echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
 
       - name: Cache npm
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache-npm
         with:
           path: ${{ steps.npm-cache.outputs.dir }}
@@ -99,7 +108,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,34 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements.txt
+    - requirements: docs/requirements.txt