From c617e4efb2426b59b26fee38ab48cebf800a2a6b Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 13:01:21 -0700 Subject: [PATCH 1/8] use virtualenv and add make lint --- .gitignore | 1 + site/Makefile | 8 ++++++++ site/dev/common.sh | 21 +++++++++++++++------ site/dev/lint.sh | 2 ++ site/dev/setup_env.sh | 2 ++ 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 3b6ac5e9a7f4..f931c10e9407 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ site/site/ site/docs/docs/ site/docs/.asf.yaml site/docs/javadoc/ +site/.venv/ # benchmark output spark/v3.4/spark/benchmark/* diff --git a/site/Makefile b/site/Makefile index 3c014f222553..14c73134f169 100755 --- a/site/Makefile +++ b/site/Makefile @@ -29,6 +29,14 @@ build: # Clean and build the docs site locally. deploy: # Clean, build, and deploy the Iceberg docs site. dev/deploy.sh $(remote_name) +.PHONY: lint +lint: # Check linting on the docs. + dev/lint.sh + +.PHONY: lint-fix +lint-fix: # Run linting with auto-fix on the docs. + dev/lint.sh --fix + .PHONY: clean clean: # Clean the local docs site. dev/clean.sh diff --git a/site/dev/common.sh b/site/dev/common.sh index 9fa14c53a395..c373c2624786 100755 --- a/site/dev/common.sh +++ b/site/dev/common.sh @@ -19,6 +19,7 @@ set -e export REMOTE="iceberg_docs" +export VENV_DIR=".venv" # Ensures the presence of a specified remote repository for documentation. # If the remote doesn't exist, it adds it using the provided URL. @@ -64,12 +65,20 @@ push_remote () { git push "${REMOTE}" "${BRANCH}" } +# Creates the virtual environment if it doesn't exist. +create_venv () { + if [ ! -d "${VENV_DIR}" ]; then + echo " --> creating virtual environment at ${VENV_DIR}" + python3 -m venv "${VENV_DIR}" + fi +} + # Installs or upgrades dependencies specified in the 'requirements.txt' file using pip. install_deps () { echo " --> install deps" - # Use pip to install or upgrade dependencies from the 'requirements.txt' file quietly - pip3 -q install -r requirements.txt --upgrade + # Use pip from venv to install or upgrade dependencies from the 'requirements.txt' file quietly + "${VENV_DIR}/bin/pip3" -q install -r requirements.txt --upgrade } # Checks if a provided argument is not empty. If empty, displays an error message and exits with a status code 1. @@ -200,8 +209,8 @@ search_exclude_versioned_docs () { cd "${ICEBERG_VERSION}/docs/" - # Modify .md files to exclude versioned documentation from search indexing - python3 -c "import os + # Modify .md files to exclude versioned documentation from search indexing using venv python + "../../${VENV_DIR}/bin/python3" -c "import os for f in filter(lambda x: x.endswith('.md'), os.listdir()): lines = open(f).readlines(); open(f, 'w').writelines(lines[:2] + ['search:\n', ' exclude: true\n'] + lines[2:]);" cd - @@ -236,7 +245,7 @@ pull_versioned_docs () { check_markdown_files () { echo " --> check markdown file styles" - if ! python3 -m pymarkdown --config markdownlint.yml scan docs/docs/nightly/docs/*.md docs/*.md README.md + if ! "${VENV_DIR}/bin/python3" -m pymarkdown --config markdownlint.yml scan docs/docs/nightly/docs/*.md docs/*.md README.md then echo "Markdown style issues found. Please run './dev/lint.sh --fix' to fix them." exit 1 @@ -245,7 +254,7 @@ check_markdown_files () { fix_markdown_files () { echo " --> fix markdown file styles" - python3 -m pymarkdown --config markdownlint.yml fix docs/docs/nightly/docs/*.md docs/*.md README.md + "${VENV_DIR}/bin/python3" -m pymarkdown --config markdownlint.yml fix docs/docs/nightly/docs/*.md docs/*.md README.md } # Cleans up artifacts and temporary files generated during documentation management. diff --git a/site/dev/lint.sh b/site/dev/lint.sh index db7762800cd4..069794578766 100755 --- a/site/dev/lint.sh +++ b/site/dev/lint.sh @@ -20,6 +20,8 @@ source dev/common.sh set -e +./dev/setup_env.sh + if [[ "$1" == "--fix" ]]; then fix_markdown_files else diff --git a/site/dev/setup_env.sh b/site/dev/setup_env.sh index f87c10d20d2e..a31d56dd5a60 100755 --- a/site/dev/setup_env.sh +++ b/site/dev/setup_env.sh @@ -22,6 +22,8 @@ set -e clean +create_venv + install_deps pull_versioned_docs From 3bdc51c03752568dfe99fb4860588f3080f757f3 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 13:17:51 -0700 Subject: [PATCH 2/8] use venv --- site/dev/build.sh | 3 ++- site/dev/deploy.sh | 3 ++- site/dev/serve.sh | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/site/dev/build.sh b/site/dev/build.sh index 67c073bc3dc7..3e491bd2eebe 100755 --- a/site/dev/build.sh +++ b/site/dev/build.sh @@ -16,10 +16,11 @@ # limitations under the License. # +source dev/common.sh set -e ./dev/setup_env.sh ./dev/lint.sh -mkdocs build +"${VENV_DIR}/bin/python3" -m mkdocs build diff --git a/site/dev/deploy.sh b/site/dev/deploy.sh index 4e59e055abc6..15bb826a4421 100755 --- a/site/dev/deploy.sh +++ b/site/dev/deploy.sh @@ -16,6 +16,7 @@ # limitations under the License. # +source dev/common.sh set -e remote_name="${1:-origin}" @@ -24,4 +25,4 @@ remote_name="${1:-origin}" echo " --> Deploy to asf-site branch of remote repository: ${remote_name}" -mkdocs gh-deploy --no-history --remote-branch=asf-site --remote-name ${remote_name} +"${VENV_DIR}/bin/python3" -m mkdocs gh-deploy --no-history --remote-branch=asf-site --remote-name ${remote_name} diff --git a/site/dev/serve.sh b/site/dev/serve.sh index 9a13dd2a1197..2be2e5f62661 100755 --- a/site/dev/serve.sh +++ b/site/dev/serve.sh @@ -16,10 +16,11 @@ # limitations under the License. # +source dev/common.sh set -e ./dev/setup_env.sh ./dev/lint.sh -mkdocs serve --dirty --watch . +"${VENV_DIR}/bin/python3" -m mkdocs serve --dirty --watch . From 6179d97f8b8738c0976105147d109433111d1d5c Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 13:27:48 -0700 Subject: [PATCH 3/8] update command --- site/dev/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/dev/common.sh b/site/dev/common.sh index c373c2624786..509dbb9d0170 100755 --- a/site/dev/common.sh +++ b/site/dev/common.sh @@ -247,7 +247,7 @@ check_markdown_files () { echo " --> check markdown file styles" if ! "${VENV_DIR}/bin/python3" -m pymarkdown --config markdownlint.yml scan docs/docs/nightly/docs/*.md docs/*.md README.md then - echo "Markdown style issues found. Please run './dev/lint.sh --fix' to fix them." + echo "Markdown style issues found. Please run 'make lint-fix' to fix them." exit 1 fi } From 94391dcffc8ab5fdcfe448e5a7189ab3b99dbc99 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 19:01:21 -0700 Subject: [PATCH 4/8] add to readme --- site/README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/site/README.md b/site/README.md index d1c42e292f06..329c667fa5be 100644 --- a/site/README.md +++ b/site/README.md @@ -76,6 +76,7 @@ The docs are built, run, and released using [make](https://www.gnu.org/software/ > help: Show help for each of the Makefile recipes. > [serve](dev/serve.sh): Clean, build, and run the site locally. > [lint](dev/lint.sh): Scan markdown files for style issues. +> [lint-fix](dev/lint.sh): Run linting with auto-fix on the markdown files. To scaffold the versioned docs and build the project, run the `build` recipe. @@ -103,9 +104,24 @@ This step will generate the staged source code which blends into the original so └─.asf.yaml ``` -It will also scan all markdown files and fail the build on any style issues. To fix style issues, run the `lint` script with fix mode. +It will also scan all markdown files and fail the build on any style issues. To fix style issues, run the `lint-fix` recipe. + +```sh +make lint-fix +``` + +### Linting + +To check for markdown style issues without building the entire site, use the `lint` recipe: + +```sh +make lint +``` + +To automatically fix markdown style issues, use the `lint-fix` recipe: + ```sh -./dev/lint.sh --fix +make lint-fix ``` From 0915efd25d397ee62ebe17d41a2fdb29071c5f38 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 19:04:05 -0700 Subject: [PATCH 5/8] run docs ci on linux and mac --- .github/workflows/docs-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index d1002f028691..67d054f8f576 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -27,7 +27,10 @@ on: jobs: build-docs: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From bfda6aa9b6e20d5f943ff9b23261cf4893f22b86 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 19:08:45 -0700 Subject: [PATCH 6/8] command --- site/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/README.md b/site/README.md index 329c667fa5be..9e9ca22f522a 100644 --- a/site/README.md +++ b/site/README.md @@ -104,7 +104,7 @@ This step will generate the staged source code which blends into the original so └─.asf.yaml ``` -It will also scan all markdown files and fail the build on any style issues. To fix style issues, run the `lint-fix` recipe. +It will also scan all markdown files and fail the build on any style issues. To fix style issues, run the `lint-fix` make command. ```sh make lint-fix @@ -112,13 +112,13 @@ make lint-fix ### Linting -To check for markdown style issues without building the entire site, use the `lint` recipe: +To check for markdown style issues without building the entire site, use the `lint` make command: ```sh make lint ``` -To automatically fix markdown style issues, use the `lint-fix` recipe: +To automatically fix markdown style issues, use the `lint-fix` make command: ```sh make lint-fix From 449d55d6b8d47ea44f2ce47a920d82db4738477d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 27 Oct 2025 19:09:43 -0700 Subject: [PATCH 7/8] style --- site/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/README.md b/site/README.md index 9e9ca22f522a..9ed5ff33655c 100644 --- a/site/README.md +++ b/site/README.md @@ -110,7 +110,7 @@ It will also scan all markdown files and fail the build on any style issues. To make lint-fix ``` -### Linting +#### Linting To check for markdown style issues without building the entire site, use the `lint` make command: From 338c24b10e58358ad2ec3e35a663c46525ba3fb6 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 2 Nov 2025 17:17:47 -0800 Subject: [PATCH 8/8] thx manu --- site/README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/site/README.md b/site/README.md index 9ed5ff33655c..aae719fcbf16 100644 --- a/site/README.md +++ b/site/README.md @@ -104,12 +104,6 @@ This step will generate the staged source code which blends into the original so └─.asf.yaml ``` -It will also scan all markdown files and fail the build on any style issues. To fix style issues, run the `lint-fix` make command. - -```sh -make lint-fix -``` - #### Linting To check for markdown style issues without building the entire site, use the `lint` make command: