diff --git a/.github/workflows/dependabot-automerge.yml b/.github/workflows/dependabot-automerge.yml index 45a0698..0732d87 100644 --- a/.github/workflows/dependabot-automerge.yml +++ b/.github/workflows/dependabot-automerge.yml @@ -13,27 +13,45 @@ concurrency: cancel-in-progress: false jobs: - automerge: - timeout-minutes: 10 - # Only run on Dependabot PRs targeting main + check-dependabot: + timeout-minutes: 5 if: > github.event.pull_request.user.login == 'dependabot[bot]' && github.event.pull_request.base.ref == 'main' runs-on: ubuntu-latest + outputs: + should_process: ${{ steps.check.outputs.should_process }} steps: - name: Fetch Dependabot metadata id: meta uses: dependabot/fetch-metadata@v2 with: - # already added to repo github-token: ${{ secrets.GH_AUTOMERGE_PAT }} - # Gate: only submodules updates; only the "upstream" submodule path + - name: Check if upstream submodule + id: check + run: | + if [[ "${{ steps.meta.outputs.package-ecosystem }}" == "submodules" ]] && \ + [[ "${{ steps.meta.outputs.dependency-names }}" == *"upstream"* ]]; then + echo "should_process=true" >> $GITHUB_OUTPUT + else + echo "should_process=false" >> $GITHUB_OUTPUT + fi + + # Generate docs after dependabot updates the submodule + generate-docs: + needs: check-dependabot + if: needs.check-dependabot.outputs.should_process == 'true' + uses: ./.github/workflows/pull-from-bazel-build.yml + secrets: inherit + + # Auto-merge after docs are generated + enable-automerge: + needs: [check-dependabot, generate-docs] + if: needs.check-dependabot.outputs.should_process == 'true' + runs-on: ubuntu-latest + steps: - name: Enable auto-merge (squash) - if: > - steps.meta.outputs.package-ecosystem == 'submodules' && - contains(steps.meta.outputs.dependency-names, 'upstream') run: gh pr merge --auto --squash "${{ github.event.pull_request.html_url }}" env: - # already added to repo - GH_TOKEN: ${{ secrets.GH_AUTOMERGE_PAT }} + GH_TOKEN: ${{ secrets.GH_AUTOMERGE_PAT }} \ No newline at end of file diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..8cda0c9 --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,20 @@ +name: Generate Documentation on PR + +on: + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + +permissions: + contents: write + pull-requests: write + +concurrency: + group: generate-docs-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + generate-docs: + # Skip Dependabot PRs (handled by auto-merge workflow) + if: github.event.pull_request.user.login != 'dependabot[bot]' + uses: ./.github/workflows/pull-from-bazel-build.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/pull-from-bazel-build.yml b/.github/workflows/pull-from-bazel-build.yml index e3838be..6674175 100644 --- a/.github/workflows/pull-from-bazel-build.yml +++ b/.github/workflows/pull-from-bazel-build.yml @@ -1,107 +1,136 @@ -name: Pull Bazel Build Upstream Repo +name: Pull Fresh Upstream Documentation on: - pull_request: - types: [opened, synchronize, reopened] - push: - branches: [main] - # called from trigger-from-bazel-repo.yml - workflow_dispatch: - inputs: - bazelCommitHash: - description: 'The commit hash of the Bazel repo to use' - type: string - default: origin/main - # allow debugging by triggering from the GitHub UI workflow_call: inputs: - bazelCommitHash: - description: 'The commit hash of the Bazel repo to use' - type: string - default: origin/main + bazelCommitHash: + description: 'Specific Bazel commit hash to checkout (optional)' + required: false + type: string + default: '' + jobs: pull-fresh-upstream: - # don't run on dependabot PRs - if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest permissions: - contents: read - packages: write + contents: write + pull-requests: write steps: - - uses: actions/checkout@v5 - with: - # Don't auto-init submodules - submodules: false + - name: Checkout repository + uses: actions/checkout@v5 + with: + # Don't auto-init submodules + submodules: false + # This token is necessary so that the action can push into the repo + # and trigger the required check (this workflow). Otherwise, the + # default GITHUB_TOKEN cannot be considered a 'synchronize' event + token: ${{ secrets.GH_AUTOMERGE_PAT }} - - name: Checkout submodules - run: git submodule update --init -- upstream + - name: Checkout submodules + run: git submodule update --init -- upstream - - name: Checkout commit of Bazel Build submodule - if: ${{ inputs.bazelCommitHash != '' }} - working-directory: upstream - run: git checkout '${{ inputs.bazelCommitHash }}' + - name: Checkout commit of Bazel Build submodule + if: ${{ inputs.bazelCommitHash != '' }} + working-directory: upstream + run: git checkout '${{ inputs.bazelCommitHash }}' - - name: Setup Bazel - uses: bazel-contrib/setup-bazel@0.15.0 - with: - bazelisk-cache: true - repository-cache: true + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.15.0 + with: + bazelisk-cache: true + repository-cache: true - - name: Build reference documentation - working-directory: upstream - run: > - bazel build - --config=docs - --build_metadata=ROLE=DOCS - --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_ORG_API_KEY }} - --bes_results_url=https://app.buildbuddy.io/invocation/ - --bes_backend=grpcs://remote.buildbuddy.io - --remote_cache=grpcs://remote.buildbuddy.io - --remote_timeout=10m - //src/main/java/com/google/devtools/build/lib:gen_reference_docs + - name: Build reference documentation + working-directory: upstream + run: > + bazel build + --config=docs + --build_metadata=ROLE=DOCS + --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_ORG_API_KEY }} + --bes_results_url=https://app.buildbuddy.io/invocation/ + --bes_backend=grpcs://remote.buildbuddy.io + --remote_cache=grpcs://remote.buildbuddy.io + --remote_timeout=10m + //src/main/java/com/google/devtools/build/lib:gen_reference_docs - # Upload reference-docs.zip as an artifact for debugging purposes - - name: Upload reference docs artifact - if: ${{ github.ref != 'refs/heads/main' }} - uses: actions/upload-artifact@v4.6.2 - with: - name: reference-docs - path: upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip - retention-days: 7 + - name: Upload reference docs artifact + if: ${{ github.ref != 'refs/heads/main' }} + uses: actions/upload-artifact@v4.6.2 + with: + name: reference-docs + path: upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip + retention-days: 7 - - name: Clean up mdx files - run: ./cleanup-mdx.sh + - name: Clean up mdx files + run: ./cleanup-mdx.sh - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version: '1.25.2' + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: '1.25.2' - - name: Initialize Go module for converter - run: | - cd html2md_converter - go mod init html-to-md-converter - go get github.com/JohannesKaufmann/html-to-markdown + - name: Initialize Go module for converter + run: | + cd html2md_converter + go mod init html-to-md-converter + go get github.com/JohannesKaufmann/html-to-markdown - - name: Build HTML to Markdown converter - run: | - cd html2md_converter - go build -o html-to-md main.go + - name: Build HTML to Markdown converter + run: | + cd html2md_converter + go build -o html-to-md main.go - - name: Convert reference documentation HTML to Markdown - run: | - # Extract and convert HTML reference docs to Markdown - ./html2md_converter/html-to-md \ - -zip upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip \ - -output reference-docs-temp + - name: Convert reference documentation HTML to Markdown + run: | + # Extract and convert HTML reference docs to Markdown + ./html2md_converter/html-to-md \ + -zip upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip \ + -output reference-docs-temp - - name: Transform upstream docs to mdx - run: ./copy-upstream-docs.sh + - name: Transform upstream docs to mdx + run: ./copy-upstream-docs.sh - - name: Create versioned navigation - run: ./docs.json.update.sh + - name: Create versioned navigation + run: ./docs.json.update.sh - - name: Clean up temporary files - run: rm -rf reference-docs-temp + - name: Clean up temporary files + run: rm -rf reference-docs-temp + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + - name: Commit and push changes + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + set -euo pipefail + + # Ensure local branch points at origin/BRANCH and is checked out + git fetch origin "$BRANCH" + git switch -C "$BRANCH" "origin/$BRANCH" + git branch --set-upstream-to="origin/$BRANCH" "$BRANCH" + + # Rebase onto latest remote before creating a new commit + git pull + + # Clean up any modified content in submodules (keep pointer changes) + # This handles the "modified content" issue while preserving submodule commit updates + cd upstream + git reset --hard + git clean -fd + cd .. + + if [[ -n "$(git status --porcelain)" ]]; then + echo "Changes detected, committing and pushing..." + git add -A + git commit -m $'chore: update documentation from upstream Bazel repo\n\nGenerated by GitHub Actions workflow from upstream Bazel repository.\nThis commit includes transformed documentation files ready for Mintlify deployment.' + + # Push back to the same branch + git push origin "HEAD:$BRANCH" + echo "Changes committed and pushed successfully" + else + echo "No changes detected, skipping commit" + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3e53677..7e4ada9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ __pycache__/ .venv/ .hugo_build.lock node_modules +# ignore go build files +html2md_converter/go.sum +html2md_converter/go.mod +html2md_converter/html-to-md diff --git a/basics/hermeticity.mdx b/basics/hermeticity.mdx index 282aad8..dcee3a9 100644 --- a/basics/hermeticity.mdx +++ b/basics/hermeticity.mdx @@ -87,7 +87,7 @@ non-hermetic actions. rich enough to allow arbitrary processing to happen in the process. You can get a log of some potentially non-hermetic actions in Bazel workspace rules by adding the flag - `--experimental_workspace_rules_log_file={{ '' }}PATH{{ '' }}` to + `--experimental_workspace_rules_log_file=PATH` to your Bazel command. Note: Make your build fully hermetic when mixing remote and local execution, diff --git a/cleanup-mdx.sh b/cleanup-mdx.sh index 31bf479..850d855 100755 --- a/cleanup-mdx.sh +++ b/cleanup-mdx.sh @@ -12,8 +12,11 @@ LOCAL_FILES=" echo "Cleaning up .mdx files..." -# Find all .mdx files in the repo (excluding upstream, work, .github, .cursor directories) -find . -name "*.mdx" -type f | while read -r mdx_file; do +# Find all .mdx files in the repo, excluding versioned directories +# This excludes any directory starting with a digit (e.g., 6.5.0, 7.6.1, 8.0.1) +find . -name "*.mdx" -type f \ + -not -path "./[0-9]*/*" \ + -not -path "./.github/*" | while read -r mdx_file; do # Check if this exact file path is in LOCAL_FILES if echo "$LOCAL_FILES" | grep -q "^\s*$mdx_file\s*$"; then diff --git a/community/roadmaps-starlark.mdx b/community/roadmaps-starlark.mdx deleted file mode 100644 index 5ce476d..0000000 --- a/community/roadmaps-starlark.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: 'Starlark Roadmap' ---- - - - -*Last verified: 2020-04-21* -([update history](https://github.com/bazelbuild/bazel-website/commits/master/roadmaps/starlark.md)) - -*Point of contact:* [laurentlb](https://github.com/laurentlb) - -## Goal - -Our goal is to make Bazel more extensible. Users should be able to easily -implement their own rules, and support new languages and tools. We want to -improve the experience of writing and maintaining those rules. - -We focus on two areas: - -* Make the language and API simple, yet powerful. -* Provide better tooling for reading, writing, updating, debugging, and testing the code. - - -## Q2 2020 - -Build health and Best practices: - -* P0. Discourage macros without have a name, and ensure the name is a unique - string literal. This work is focused on Google codebase, but may impact - tooling available publicly. -* P0. Make Buildozer commands reliable with regard to selects and variables. -* P1. Make Buildifier remove duplicates in lists that we don’t sort because of - comments. -* P1. Update Buildifier linter to recommend inlining trivial expressions. -* P2. Study use cases for native.existing_rule[s]() and propose alternatives. -* P2. Study use cases for the prelude file and propose alternatives. - -Performance: - -* P1. Optimize the Starlark interpreter using flat environments and bytecode - compilation. - -Technical debt reduction: - -* P0. Add ability to port native symbols to Starlark underneath @bazel_tools. -* P1. Delete obsolete flags (some of them are still used at Google, so we need to - clean the codebase first): `incompatible_always_check_depset_elements`, - `incompatible_disable_deprecated_attr_params`, - `incompatible_no_support_tools_in_action_inputs`, `incompatible_new_actions_api`. -* P1. Ensure the followin flags can be flipped in Bazel 4.0: - `incompatible_disable_depset_items`, `incompatible_no_implicit_file_export`, - `incompatible_run_shell_command_string`, - `incompatible_restrict_string_escapes`. -* P1. Finish lib.syntax work (API cleanup, separation from Bazel). -* P2. Reduce by 50% the build+test latency of a trivial edit to Bazel’s Java packages. - -Community: - -* `rules_python` is active and well-maintained by the community. -* Continuous support for rules_jvm_external (no outstanding pull requests, issue - triage, making releases). -* Maintain Bazel documentation infrastructure: centralize and canonicalize CSS - styles across bazel-website, bazel-blog, docs -* Bazel docs: add CI tests for e2e doc site build to prevent regressions. - -## Q1 2020 - -Build health and Best practices: - -* Allow targets to track their macro call stack, for exporting via `bazel query` -* Implement `--incompatible_no_implicit_file_export` -* Remove the deprecated depset APIs (#5817, #10313, #9017). -* Add a cross file analyzer in Buildifier, implement a check for deprecated - functions. - -Performance: - -* Make Bazel’s own Java-based tests 2x faster. -* Implement a Starlark CPU profiler. - -Technical debt reduction: - -* Remove 8 incompatible flags (after flipping them). -* Finish lib.syntax cleanup work (break dependencies). -* Starlark optimization: flat environment, bytecode compilation -* Delete all serialization from analysis phase, if possible -* Make a plan for simplifying/optimizing lib.packages - -Community: - -* Publish a Glossary containing definitions for all the Bazel-specific terms diff --git a/community/update.mdx b/community/update.mdx deleted file mode 100644 index be0e07d..0000000 --- a/community/update.mdx +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: 'Community updates' ---- - - - -Join Bazel developer relations engineers for the monthly community update -livestream, or catch up on past ones. - -Title | Date | Description | Speakers --------- | -------- | -------- | -------- -[Roadmap Introduction](https://www.youtube.com/watch?v=gYrZDl7K9JM) | 5/19/2022 | The inaugural Bazel Community Update, introducing the community to some of Google's Bazel leadership to talk about the general state of the project and its upcoming roadmap | Sven Tiffe, Tony Aiuto, Radhika Advani -[Hands-On with Bzlmod](https://www.youtube.com/watch?v=MuW5XNcFukE) | 6/23/2022 | This month, we're joined by Google engineers Yun Peng and Xudong Yang to talk about Bzlmod, the new dependency system that is expected to go GA later this year. We'll cover the motivation behind the change, the new capabilities it brings to the table, and walk through some examples of it in action. | Yun Peng, Xudong Yang -[Extending Gazelle to generate BUILD files](https://www.youtube.com/watch?v=E1-U7EAfhXw) | 7/21/2022 | This month we're joined by Son Luong Ngoc who will be showing the Gazelle language extension system. We'll briefly touch on how it works under the covers, existing extensions, and how to go about writing your own extensions to ease the migration to Bazel. | Son Luong Ngoc -[Using Bazel for JavaScript Projects](https://www.youtube.com/watch?v=RIfYqX0JJYk) | 8/18/2022 | In this update, Alex Eagle joins us to talk about running JavaScript build tooling under Bazel. We'll look at a couple of examples: a Vue.js frontend and Nest backend. We'll cover the migration to newer rules_js provided by Aspect, and study how the tooling allows for fetching third-party dependencies and resolving them in the Node.js runtime. | Alex Eagle -[Like Peanut Butter & Jelly: Integrating Bazel with JetBrains IntelliJ](https://www.youtube.com/watch?v=wMrua-W-LC4) | 9/15/2022 | Bazel is awesome. IntelliJ is awesome. Naturally, they are more awesome together. Bazel IntelliJ plugin gurus Mai Hussien from Google and Justin Kaeser from JetBrains join us this month to give a live demo and walkthrough of the plugin's capabilities. Both new and experienced plugin users are welcome to come with questions. | Mai Hussien, Justin Kaeser -[Bazel at scale for surgical robots](https://www.youtube.com/watch?v=kCs1xa45yjM) | 10/27/2022 | What do you do when CMake CI runs for four hours? Join Guillaume Maudoux of Tweag to learn about how they migrated large, embedded robotic applications to Bazel. Topics include configuring toolchains for cross compilation, improving CI performance, managing third-party dependencies, and creating a positive developer experience — everything needed to ensure that Bazel lives up to “{Fast, Correct} — Choose Two”. | Guillaume Maudoux -[The Ghosts of Bazel Past, Present, and Future](https://www.youtube.com/watch?v=uRjSghJQlsw) | 12/22/2022 | For our special holiday Community Update and last of 2022, I'll be joined by Google's Sven Tiffe and Radhika Advani where we'll be visited by the ghosts of Bazel Past (2022 year in review), Present (Bazel 6.0 release), and Future (what to expect in 2023). | Sven Tiffe, Radhika Advani diff --git a/community/users.mdx b/community/users.mdx new file mode 100644 index 0000000..91e26c4 --- /dev/null +++ b/community/users.mdx @@ -0,0 +1,750 @@ +--- +title: 'Who''s Using Bazel' +--- + + + +Note: Using Bazel? You can add your company on +[StackShare](https://stackshare.io/bazel). To add yourself to this page, +contact [product@bazel.build](mailto:product@bazel.build). + +This page lists companies and OSS projects that are known to use Bazel. +This does not constitute an endorsement. + +## Companies using Bazel + +### [acqio](https://acqio.com.br) + + + +Acqio is a Fintech that provides payment products and services for small and +medium merchants. Acqio has a handful of monorepos and uses Bazel along with +Kubernetes to deliver fast and reliable microservices. + +### [Adobe](https://www.adobe.com/) + + + +Adobe has released Bazel [rules](https://github.com/adobe/rules_gitops) for +continuous, GitOps driven Kubernetes deployments. + +### [Asana](https://asana.com) + + + +Asana is a web and mobile application designed to help teams track their work. +In their own words: + +> Bazel has increased reliability, stability, and speed for all of builds/tests +at Asana. We no longer need to clean because of incorrect caches. + +### [Ascend.io](https://ascend.io) + +Ascend is a Palo Alto startup that offers solutions for large data sets +analysis. Their motto is _Big data is hard. We make it easy_. + +### [ASML](https://asml.com) + + + +ASML is an innovation leader in the semiconductor industry. We provide chipmakers +with everything they need – hardware, software and services – to mass produce +patterns on silicon through lithography. + +### [Beeswax](https://www.beeswax.com/) + +> Beeswax is a New York based startup that provides real time bidding as +service. Bazel powers their Jenkins based continuous integration and deployment +framework. Beeswax loves Bazel because it is blazingly fast, correct and well +supported across many languages and platforms. + +### [Braintree](https://www.braintreepayments.com) + + + +Braintree, a PayPal subsidiary, develops payment solutions for websites and +applications. They use Bazel for parts of their internal build and Paul Gross +even posted a +[nice piece about how their switch to Bazel went](https://www.pgrs.net/2015/09/01/migrating-from-gradle-to-bazel/). + +### [Canva](https://www.canva.com/) + + +Canva leverages Bazel to manage its large polyglot codebase, which includes +Java, TypeScript, Scala, Python, and more. Migration to Bazel has delivered +significant developer and compute infrastructure efficiencies, for example 5-6x +decreases in average CI build times, and it continues to become the foundation +of fast, reproducible, and standardised software builds at the company. + +### [CarGurus](https://www.cargurus.com) + + +CarGurus is on a mission to build the world's most trusted and transparent +automotive marketplace and uses Bazel to build their polyglot monorepo. + +### [Compass](https://www.compass.com) + +Compass is a tech-driven real estate platform. With an elite team of real +estate, technology and business professionals, we aim to be the best and most +trusted source for home seekers. + +### [Databricks](https://databricks.com) + + +Databricks provides cloud-based integrated workspaces based on Apache Spark™. + +> The Databricks codebase is a Monorepo, containing the Scala code that powers +most of our services, Javascript for front-end UI, Python for scripting, +Jsonnet to configure our infrastructure, and much more [...] Even though our +monorepo contains a million lines of Scala, working with code within is fast +and snappy. +([Speedy Scala Builds with Bazel at Databricks](https://databricks.com/blog/2019/02/27/speedy-scala-builds-with-bazel-at-databricks.html)) + +### [Dataform](https://dataform.co) + +Dataform provides scalable analytics for data teams. They maintain a handful of +NPM packages and a documentation site in one single monorepo and they do it all +with Bazel. + +After the migration to Bazel, they +[reported many benefits](https://github.com/bazelbuild/rules_nodejs#user-testimonials), +including: + +> * Faster CI: we enabled the remote build caching which has reduced our average build time from 30 minutes to 5 (for the entire repository). +> * Improvements to local development: no more random bash scripts that you forget to run, incremental builds reduced to seconds from minutes +> * Developer setup time: New engineers can build all our code with just 3 dependencies - bazel, docker and the JVM. The last engineer to join our team managed to build all our code in < 30 minutes on a brand new, empty laptop + +### [Deep Silver FISHLABS](https://www.dsfishlabs.com) +Deep Silver FISHLABS is a developer of high-end 3D games. They use Bazel with +C++/Python/Go/C as a base for their internal build tooling and especially for +baking and deploying all their 3D Assets. + +### [Dropbox](https://www.dropbox.com/) + +At Dropbox, Bazel is a key component to our distributed build and test +environment. We use Bazel to combine TypeScript/Python/Go/C/Rust into reliable +production releases. + +### [Engel & Völkers](https://www.engelvoelkers.com) + +Engel & Völkers AG is a privately owned German company that, via a series of +franchised offices, provides services related to real estate transactions. + +> One of our internal project has seen a decrease of compilation time from 11 +minutes to roughly 1 minute, this was an impressive achievement and we are +currently working on bringing Bazel to more projects. +([Experimenting with Google Cloud Build and Bazel](https://www.engelvoelkers.com/en/tech/engineering/software-engineering/experimenting-with-google-cloud-build-and-bazel/)) + +### [Etsy](https://www.etsy.com/) + + +Etsy is an e-commerce website focused on handmade or vintage items and supplies, +as well as unique factory-manufactured items. + +They use Bazel to build and test its Java-based search platform. Bazel produces +both packages for bare metal servers and repeatable Docker images. + +### [Evertz.io](https://www.evertz.io/) + +Evertz.io is a multi-tenant, serverless SaaS platform for offering cost +effective, multi-regional services worldwide to the Broadcast Media Industry, +created by [Evertz Microsystems](https://en.wikipedia.org/wiki/Evertz_Microsystems). + +The website is fully built and deployed with an Angular and Bazel workflow +([source](https://twitter.com/MattMackay/status/1113947685508341762)). + +### [FINDMINE](http://www.findmine.com) + + +FINDMINE is a automation technology for the retail industry that uses machine +learning to scale the currently manual and tedious process of product curation. +We use Bazel to mechanize our entire python package building, testing, and +deployment process. + +### [Flexport](https://www.flexport.com/) + +Flexport is a tech-enabled global freight forwarder; our mission is to make +global trade easier for everyone. At Flexport, we use Bazel to build/test our +Java/JavaScript services and client libraries and to generate Java and Ruby +code from protobuf definitions. +[Read about how we run individual JUnit 5 tests in isolation with Bazel.](https://flexport.engineering/connecting-bazel-and-junit5-by-transforming-arguments-46440c6ea068) + +### [Foursquare](https://foursquare.com) + + +Foursquare's mission is to create technology that constructs meaningful +bridges between digital spaces and physical places. We manage millions of +lines of primarily Scala and Python code powering data-intensive +applications, including complex codegen and container build processes, with +Bazel. + +### [GermanTechJobs](https://germantechjobs.de) + + +Bazel has simplified our workflows 10-fold and enabled shipping features at +scale. + +### [Google](https://google.com) + + +Bazel was designed to be able to scale to Google's needs and meet Google's +requirements of reproducibility and platform/language support. All software at +Google is built using Bazel. Google uses Bazel and its rules for millions of +builds every day. + +### [Huawei](http://www.huawei.com/) + +> Huawei Technologies is using Bazel in about 30 projects, they are Java/Scala/Go +projects, except for Go projects, others originally were built by Maven. We +write a simple tool to translate a Maven-built project into Bazel-built one. +More and more projects will use Bazel in recent future. + +### [IMC Trading](https://imc.com) + + +> IMC is a global proprietary trading firm and market maker headquarted in +Amsterdam. We are using Bazel to continuously build and test our +Java/C++/Python/SystemVerilog projects. + +### [Improbable.io](https://improbable.io/) + +Improbable.io develops SpatialOS, a distributed operating system that enables +creating huge simulations inhabited by millions of complex entities. + +### [Interaxon](https://www.choosemuse.com/) + +InteraXon is a thought-controlled computing firm that creates hardware and +software platforms to convert brainwaves into digital signals. + +### [Jupiter](https://jupiter.co/) + +Jupiter is a company that provides delivery of groceries and household +essentials every week. + +They use Bazel in their backend code, specifically to compile protos and Kotlin +to JVM binaries, using remote caching. +([source](https://starship.jupiter.co/jupiter-stack/)) + +### [Just](https://gojust.com/) + +Just is an enterprise financial technology company, headquartered in Norway, +creating software solutions to transform how global corporate treasurers manage +risk and liquidity. Their entire application stack is built with Bazel. + +### [Line](https://line.me/) + +Line provides an app for instant communications, which is the most popular +messaging application in Japan. +They use Bazel on their codebase consisting of about 60% Swift and 40% +C/C++/Objective-C/Objective-C++ +([source](https://twitter.com/thi_dt/status/1253334262020886532)). + +> After switching to Bazel, we were able to achieve a huge improvement in the +build times. This brought a significant improvement in the turn-around time +during a QA period. Distributing a new build to our testers no longer means +another hour waiting for building and testing. +([Improving Build Performance of LINE for iOS with Bazel](https://engineering.linecorp.com/en/blog/improving-build-performance-line-ios-bazel/)) + +### [LingoChamp](https://www.liulishuo.com/en) + + +LingoChamp provides professional solutions to English learners. We use Bazel +for our go, java and python projects. + +### [LinkedIn](https://linkedin.com/) + + +LinkedIn, a subsidiary of Microsoft, is the world’s largest professional social +network. LinkedIn uses Bazel for building its iOS Apps. + +### [Lucid Software](https://lucid.co/) + + + +Lucid Software is a leader in visual collaboration, helping teams see and build the +future from idea to reality. With its products—[Lucidchart](https://www.lucidchart.com/), +[Lucidspark](https://lucidspark.com/), and [Lucidscale](https://lucidscale.com/)—teams +can align around a shared vision, clarify complexity, and collaborate visually, no +matter where they’re located. + +Lucid uses Bazel to build millions of lines of Scala and TypeScript. +Migrating to Bazel has tremendously sped up its builds, reduced external +dependencies on the build environment, and simplified developers' experience +with the build system. Bazel has improved developer productivity at Lucid and +unlocked further growth. + +### [Lyft](https://www.lyft.com/) + +Lyft is using Bazel for their iOS ([source](https://twitter.com/SmileyKeith/status/1116486751806033920)) and Android Apps. + +### [Meetup](http://www.meetup.com/) + +Meetup is an online social networking portal that facilitates offline group +meetings. +The Meetup engineering team contributes to +[rules_scala](https://github.com/bazelbuild/rules_scala) and is the +maintainer of [rules_avro](https://github.com/meetup/rules_avro) +and [rules_openapi](https://github.com/meetup/rules_openapi). + + +### [Nvidia](https://www.nvidia.com/) + +> At Nvidia we have been using dazel(docker bazel) for python to work around +some of bazel's python short comings. Everything else runs in normal bazel +(Mostly Go / Scala/ C++/ Cuda) +([source](https://twitter.com/rwhitcomb/status/1080887723433447424)) + + +### [Peloton Technology](http://www.peloton-tech.com) + +Peloton Technology is an automated vehicle technology company that tackles truck +accidents and fuel use. They use Bazel to _enable reliable builds for automotive +safety systems_. + +### [Pigweed](https://pigweed.dev) + + + +Pigweed is an open-source solution for sustained, robust, and rapid embedded +product development for large teams. Pigweed has shipped in millions of +devices, including Google's suite of Pixel devices, Nest thermostats, +[satellites](https://www.spinlaunch.com/), and [autonomous aerial +drones](https://www.flyzipline.com/). + +Pigweed [uses Bazel as its primary build +system](https://pigweed.dev/seed/0111-build-systems.html). The [Bazel for +Embedded][pw-bazel-great] blog post discusses why we think it's a great build +system for embedded projects! + +[pw-bazel-great]: https://blog.bazel.build/2024/08/08/bazel-for-embedded.html#why-bazel-for-embedded + +### [Pinterest](https://www.pinterest.com/) + + + +Pinterest is the world’s catalog of ideas. They use Bazel to build various +backend services (Java/C++) and the iOS application (Objective-C/C++). + +> We identified Bazel was the best fit for our goals to build a foundation for +an order of magnitude improvement in performance, eliminate variability in +build environments and adopt incrementally. As a result, we’re now shipping all +our iOS releases using Bazel. +[Developing fast & reliable iOS builds at Pinterest](https://medium.com/@Pinterest_Engineering/developing-fast-reliable-ios-builds-at-pinterest-part-one-cb1810407b92) + +### [PubRef](https://github.com/pubref) + +PubRef is an emerging scientific publishing platform. They use Bazel with +[rules_closure](https://github.com/bazelbuild/rules_closure) to build the +frontend, native java rules to build the main backend, +[rules_go](https://github.com/bazelbuild/rules_go), +[rules_node](https://github.com/pubref/rules_node), and +[rules_kotlin](https://github.com/pubref/rules_kotlin) to build assorted +backend services. [rules_protobuf](https://github.com/pubref/rules_protobuf) is +used to assist with gRPC-based communication between backend services. +PubRef.org is based in Boulder, CO. + +### [Redfin](https://redfin.com/) +Redfin is a next-generation real estate brokerage with full-service local +agents. They use Bazel to build and deploy the website and various backend +services. + +> With the conversion mostly behind us, things are greatly improved! Our CI +builds are faster (*way* faster: they used to take 40–90 minutes, and now dev +builds average 5–6 minutes). Reliability is far higher, too. This is harder to +quantify, but the shift from unexplained build failures being something that +“just happens” to being viewed as real problems to be solved has put us on a +virtuous cycle of ever-increasing reliability. +([We Switched from Maven to Bazel and Builds Got 10x Faster](https://redfin.engineering/we-switched-from-maven-to-bazel-and-builds-got-10x-faster-b265a7845854)) + +### [Ritual](https://ritual.co) + + +Ritual is a mobile pick up app, connecting restaurants with customers to offer +a simple, time-saving tool to get the food and beverages they want, without the +wait. Ritual uses Bazel for their backend services. + +### [Snap](https://www.snap.com/en-US/) + +Snap, the developer of Snapchat messaging app, has migrated from Buck to Bazel +in 2020 ([source](https://twitter.com/wew/status/1326957862816509953)). For more +details about their process, see their [engineering blog](https://eng.snap.com/blog/). + +### [Stripe](https://stripe.com) + + +Stripe provides mobile payment solutions. They use Bazel in their build and test pipelines, as detailed in their [engineering blog](https://stripe.com/blog/fast-secure-builds-choose-two). + +### [Tinder](https://tinder.com) + + +Tinder migrated its iOS app from CocoaPods to Bazel +in 2021 ([source](https://medium.com/tinder/bazel-hermetic-toolchain-and-tooling-migration-c244dc0d3ae)). + +### [Tink](https://tink.com/) + + +Tink is a european fintech, building the best way to connect to banks across +Europe. + +They are using Bazel to build their backend services from a polyglot monorepo. +Engineers at Tink are organizing the [bazel build //stockholm/...](https://www.meetup.com/BazelSTHLM/) +meetup group. + +### [Tokopedia](https://www.tokopedia.com/) + +Tokopedia is an Indonesian technology company specializing in e-commerce, with +over 90 million monthly active users and over 7 million merchants on the +platform. + +They wrote the article +[How Tokopedia Achieved 1000% Faster iOS Build Time](https://medium.com/tokopedia-engineering/how-tokopedia-achieved-1000-faster-ios-build-time-7664b2d8ae5), +where they explain how Bazel sped up their builds. The build duration went from +55 minutes to 10 minutes by using Bazel, and down to 5 minutes with remote +caching. + +### [Trunk.io](https://trunk.io/merge/trunk-merge-and-bazel) + + +Trunk is a San Francisco-based company backed by Andreessen Horowitz and Initialized Capital. Trunk offers a powerful pull request merge service with first-class support for the Bazel build system. By leveraging Bazel's understanding of dependencies within a codebase, Trunk's merge service intelligently creates parallel merge lanes, allowing independent changes to be tested and merged simultaneously. + +> Trunk’s internal monorepo builds modern C++ 20 and typescript all while leveraging bazel graph knowledge to selectively test and merge code. + +### [Twitter](https://twitter.com/) + +Twitter has made the decision to migrate from Pants to Bazel as their primary +build tool +([source](https://groups.google.com/forum/#!msg/pants-devel/PHVIbVDLhx8/LpSKIP5cAwAJ)). + +### [Two Sigma](https://www.twosigma.com/) + + +Two Sigma is a New York-headquartered technology company dedicated to finding +value in the world’s data. + +### [TypeDB](https://typedb.com) +TypeDB Logo + +TypeDB is a database technology that can be used to intuitively model +interconnected data. Through its type-theoretic and polymorphic query language, +TypeQL, the data can be accessed with simple, human-readable queries that run at +lightspeed. + +Bazel enables the TypeDB team to build a highly-orchestrated CI and distribution +pipeline that manages many repositories in a wide variety of languages, and +deploys to numerous platforms seamlessly. The TypeDB team has also released +Bazel rules for assembling and deploying software distributions. + +### [Uber](https://www.uber.com) + +Uber is a ride-hailing company. With 900 active developers, Uber’s Go monorepo +is likely one of the largest Go repositories using Bazel. See the article +[Building Uber’s Go Monorepo with Bazel](https://eng.uber.com/go-monorepo-bazel/) +to learn more about their experience. + +### [Uber Advanced Technologies Group](https://www.uber.com/info/atg/) + + +Uber Advanced Technologies Group is focused on autonomous vehicle efforts at +Uber, including trucking/freight and autonomous ride sharing. The organization +uses Bazel as its primary build system. + +### [Vistar Media](http://vistarmedia.com) +Vistar Media is an advertising platform that enables brands to reach consumers +based on their behavior in the physical world. Their engineering team is +primarily based out of Philadelphia and is using Bazel for builds, deploys, to +speed up testing, and to consolidate repositories written with a variety of +different technologies. + +### [VMware](https://www.vmware.com/) +VMware uses Bazel to produce deterministic, reliable builds while developing +innovative products for their customers. + +### [Wix](https://www.wix.com/) + +Wix is a cloud-based web development platform. Their backend uses Java and Scala +code. They use remote execution with Google Cloud Build. + +> We have seen about 5 times faster clean builds when running with bazel remote +execution which utilizes bazel’s great build/test parallelism capabilities when +it dispatches build/test actions to a worker farm. Average build times are more +than 10 times faster due to the utilization of bazel’s aggressive caching +mechanism. +([Migrating to Bazel from Maven or Gradle? 5 crucial questions you should ask yourself](https://medium.com/wix-engineering/migrating-to-bazel-from-maven-or-gradle-5-crucial-questions-you-should-ask-yourself-f23ac6bca070)) + +### [Zenly](https://zen.ly/) + +Zenly is a live map of your friends and family. It’s the most fun way to meet up +— or just see what’s up! — so you can feel together, even when you're apart. + + +*** + +## Open source projects using Bazel + +### [Abseil](https://abseil.io/) + +Abseil is an open-source collection of C++ code (compliant to C++11) designed +to augment the C++ standard library. + +### [Angular](https://angular.io) + + + +Angular is a popular web framework. +Angular is [built with Bazel](https://github.com/angular/angular/blob/master/docs/BAZEL.md). + +### [Apollo](https://github.com/ApolloAuto/apollo) + +Apollo is a high performance, flexible architecture which accelerates the +development, testing, and deployment of Autonomous Vehicles. + +### [brpc](https://github.com/brpc/brpc) + +An industrial-grade RPC framework used throughout Baidu, with 1,000,000+ +instances(not counting clients) and thousands kinds of services, called +"baidu-rpc" inside Baidu. + +### [cert-manager](https://github.com/jetstack/cert-manager) + +cert-manager is a Kubernetes add-on to automate the management and issuance of +TLS certificates from various issuing sources. It will ensure certificates are +valid and up to date periodically, and attempt to renew certificates at an +appropriate time before expiry. + +### [CallBuilder](https://github.com/google/CallBuilder) + +A Java code generator that allows you to create a builder by writing one +function. + +### [CPPItertools](https://github.com/ryanhaining/cppitertools) + +C++ library providing range-based for loop add-ons inspired by the Python +builtins and itertools library. Like itertools and the Python3 builtins, this +library uses lazy evaluation wherever possible. + +### [Copybara](https://github.com/google/copybara) + +Copybara is a tool for transforming and moving code between repositories. + +### [Dagger](https://google.github.io/dagger/) + +Dagger is a fully static, compile-time dependency injection framework for both +Java and Android. + +### [DAML](https://github.com/digital-asset/daml) + +DAML is a smart contract language for building future-proof distributed +applications on a safe, privacy-aware runtime. + +### [DeepMind Lab](https://github.com/deepmind/lab) + +A customisable 3D platform for agent-based AI research. + +### [Drake](https://github.com/RobotLocomotion/drake) + +Drake is a C++ toolbox started at MIT and now led by the Toyota Research +Institute. It is a collection of tools for analyzing the dynamics of our robots +and building control systems for them, with a heavy emphasis on +optimization-based design/analysis. + +### [Envoy](https://github.com/lyft/envoy) + +C++ L7 proxy and communication bus + +### [Error Prone](https://github.com/google/error-prone) + +Catches common Java mistakes as compile-time errors. (Migration to Bazel is in +progress.) + +### [Extensible Service Proxy](https://github.com/cloudendpoints/esp) + +Extensible Service Proxy, a.k.a. ESP is a proxy which enables API management +capabilities for JSON/REST or gRPC API services. The current implementation is +based on an NGINX HTTP reverse proxy server. + +### [FFruit](https://gitlab.com/perezd/ffruit/) + +FFruit is a free & open source Android application to the popular service +[Falling Fruit](https://fallingfruit.org). + +### [Gerrit Code Review](https://gerritcodereview.com) + +Gerrit is a code review and project management tool for Git based projects. + +### [Gitiles](https://gerrit.googlesource.com/gitiles/) + +Gitiles is a simple repository browser for Git repositories, built on JGit. + +### [Grakn](https://github.com/graknlabs/grakn) + +Grakn (https://grakn.ai/) is the knowledge graph engine to organise complex +networks of data and make it queryable. + +### [GRPC](http://www.grpc.io) +A language-and-platform-neutral remote procedure call system. +(Bazel is a supported, although not primary, build system.) + +### [gVisor](https://github.com/google/gvisor) +gVisor is a container runtime sandbox. + +### [Guetzli](https://github.com/google/guetzli/) + +Guetzli is a JPEG encoder that aims for excellent compression density at high +visual quality. + +### [Gulava](http://www.github.com/google/gulava/) + +A Java code generator that lets you write Prolog-style predicates and use them +seamlessly from normal Java code. + +### [Heron](https://github.com/apache/incubator-heron) + +Heron is a realtime, distributed, fault-tolerant stream processing engine from +Twitter. + +### [Internet Computer Protocol](https://internetcomputer.org/) + + + +The Internet Computer Protocol is a publicly available blockchain network that +enables replicated execution of general-purpose computation, serving hundreds +of thousands of applications and their users. + +### [Jazzer](https://github.com/CodeIntelligenceTesting/jazzer) + + + +Jazzer is a fuzzer for Java and other JVM-based languages that integrates with JUnit 5. + +### [JGit](https://eclipse.org/jgit/) + +JGit is a lightweight, pure Java library implementing the Git version control +system. + +### [Jsonnet](https://jsonnet.org/) + +An elegant, formally-specified config generation language for JSON. +(Bazel is a supported build system.) + +### [Kubernetes](https://github.com/kubernetes/kubernetes) + + +Kubernetes is an open source system for managing containerized applications +across multiple hosts, providing basic mechanisms for deployment, maintenance, +and scaling of applications. + +### [Kythe](https://github.com/google/kythe) + +An ecosystem for building tools that work with code. + +### [ls-lint](https://github.com/loeffel-io/ls-lint) + + + +An extremely fast directory and filename linter - Bring some structure to your +project file system. + +### [Nomulus](https://github.com/google/nomulus) + +Top-level domain name registry service on Google App Engine. + +### [ONOS : Open Network Operating System](https://github.com/opennetworkinglab/onos) + + +ONOS is the only SDN controller platform that supports the transition from +legacy “brown field” networks to SDN “green field” networks. This enables +exciting new capabilities, and disruptive deployment and operational cost points +for network operators. + +### [PetitParser for Java](https://github.com/petitparser/java-petitparser) + +Grammars for programming languages are traditionally specified statically. +They are hard to compose and reuse due to ambiguities that inevitably arise. +PetitParser combines ideas from scannnerless parsing, parser combinators, +parsing expression grammars and packrat parsers to model grammars and parsers +as objects that can be reconfigured dynamically. + +### [PlaidML](https://github.com/plaidml/plaidml) + +PlaidML is a framework for making deep learning work everywhere. + +### [Project V](https://www.v2ray.com/) + + +Project V is a set of tools to help you build your own privacy network over +internet. + +### [Prysmatic Labs Ethereum 2.0 Implementation](https://github.com/prysmaticlabs/prysm) + +Prysm is a sharding client for Ethereum 2.0, a blockchain-based distributed +computing platform. + +### [Ray](https://github.com/ray-project/ray) + +Ray is a flexible, high-performance distributed execution framework. + +### [Resty](https://github.com/go-resty/resty) + +Resty is a Simple HTTP and REST client library for Go (inspired by Ruby +rest-client). + +### [Roughtime](https://roughtime.googlesource.com/roughtime) + +Roughtime is a project that aims to provide secure time synchronisation. + +### [Selenium](https://github.com/SeleniumHQ/selenium) + +Selenium is a portable framework for testing web applications. + +### [Semantic](https://github.com/github/semantic) + +Semantic is a Haskell library and command line tool for parsing, analyzing, and +comparing source code. It is developed by GitHub (and used for example for the +code navigation). + +### [Served](https://github.com/meltwater/served) + +Served is a C++ library for building high performance RESTful web servers. + +### [Sonnet](https://github.com/deepmind/sonnet) + +Sonnet is a library built on top of TensorFlow for building complex neural +networks. + +### [Sorbet](https://github.com/sorbet/sorbet) + +Sorbet is a fast, powerful type checker for a subset of Ruby. It scales to +codebases with millions of lines of code and can be adopted incrementally. + +### [Spotify](https://spotify.com) + +Spotify is using Bazel to build their iOS and Android Apps ([source](https://twitter.com/BalestraPatrick/status/1573355078995566594)). + +### [Tink](https://github.com/google/tink) + +Tink is a multi-language, cross-platform, open source library that provides +cryptographic APIs that are secure, easy to use correctly, and hard(er) to +misuse. + +### [TensorFlow](http://tensorflow.org) + + +An open source software library for machine intelligence. + +### [Turbo Santa](https://github.com/turbo-santa/turbo-santa-common) + +A platform-independent GameBoy emulator. + +### [Wycheproof](https://github.com/google/wycheproof) + +Project Wycheproof tests crypto libraries against known attacks. + +### [XIOSim](https://github.com/s-kanev/XIOSim) + +XIOSim is a detailed user-mode microarchitectural simulator for the x86 +architecture. + +### [ZhihuDailyPurify](https://github.com/izzyleung/ZhihuDailyPurify) + +ZhihuDailyPurify is a light weight version of Zhihu Daily, a Chinese +question-and-answer webs. diff --git a/configure/best-practices.mdx b/configure/best-practices.mdx index 00b4391..deecf9d 100644 --- a/configure/best-practices.mdx +++ b/configure/best-practices.mdx @@ -67,7 +67,7 @@ then the `BUILD` files are misleading. ## Using the `.bazelrc` file For project-specific options, use the configuration file your -`{{ '' }}workspace{{ '' }}/.bazelrc` (see [bazelrc format](/run/bazelrc)). +`workspace/.bazelrc` (see [bazelrc format](/run/bazelrc)). If you want to support per-user options for your project that you **do not** want to check into source control, include the line: @@ -75,11 +75,12 @@ want to check into source control, include the line: ``` try-import %workspace%/user.bazelrc ``` -(or any other file name) in your `{{ '' }}workspace{{ '' }}/.bazelrc` +(or any other file name) in your `workspace/.bazelrc` and add `user.bazelrc` to your `.gitignore`. The open-source [bazelrc-preset.bzl](https://github.com/bazel-contrib/bazelrc-preset.bzl) + generates a custom bazelrc file that matches your Bazel version and provides a preset of recommended flags. diff --git a/configure/windows.mdx b/configure/windows.mdx new file mode 100644 index 0000000..2dbb90e --- /dev/null +++ b/configure/windows.mdx @@ -0,0 +1,317 @@ +--- +title: 'Using Bazel on Windows' +--- + + + +This page covers Best Practices for using Bazel on Windows. For installation +instructions, see [Install Bazel on Windows](/install/windows). + +## Known issues + +Windows-related Bazel issues are marked with the "area-Windows" label on GitHub. +[GitHub-Windows]. + +[GitHub-Windows]: https://github.com/bazelbuild/bazel/issues?q=is%3Aopen+is%3Aissue+label%3Aarea-Windows + +## Best practices + +### Avoid long path issues + +Some tools have the [Maximum Path Length Limitation](https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation) on Windows, including the MSVC compiler. +To avoid hitting this issue, you can specify a short output directory for Bazel by the [\-\-output_user_root](/reference/command-line-reference#flag--output_user_root) flag. + +For example, add the following line to your bazelrc file: + +```none +startup --output_user_root=C:/tmp +``` + +### Enable symlink support + +Some features require Bazel to be able to create file symlinks on Windows, +either by enabling +[Developer Mode](https://docs.microsoft.com/en-us/windows/uwp/get-started/enable-your-device-for-development) +(on Windows 10 version 1703 or newer), or by running Bazel as an administrator. +This enables the following features: + +* [\-\-windows_enable_symlinks](/reference/command-line-reference#flag--windows_enable_symlinks) +* [\-\-enable_runfiles](/reference/command-line-reference#flag--enable_runfiles) + +To make it easier, add the following lines to your bazelrc file: + +```none +startup --windows_enable_symlinks + +build --enable_runfiles +``` + +**Note**: Creating symlinks on Windows is an expensive operation. The `--enable_runfiles` flag can potentially create a large amount of file symlinks. Only enable this feature when you need it. + +{/* TODO(pcloudy): https://github.com/bazelbuild/bazel/issues/6402 + Write a doc about runfiles library and add a link to it here */} + +### Running Bazel: MSYS2 shell vs. command prompt vs. PowerShell + +**Recommendation:** Run Bazel from the command prompt (`cmd.exe`) or from +PowerShell. + +As of 2020-01-15, **do not** run Bazel from `bash` -- either +from MSYS2 shell, or Git Bash, or Cygwin, or any other Bash variant. While Bazel +may work for most use cases, some things are broken, like +[interrupting the build with Ctrl+C from MSYS2](https://github.com/bazelbuild/bazel/issues/10573)). +Also, if you choose to run under MSYS2, you need to disable MSYS2's +automatic path conversion, otherwise MSYS will convert command line arguments +that _look like_ Unix paths (such as `//foo:bar`) into Windows paths. See +[this StackOverflow answer](https://stackoverflow.com/a/49004265/7778502) +for details. + +### Using Bazel without Bash (MSYS2) + +#### Using bazel build without Bash + +Bazel versions before 1.0 used to require Bash to build some rules. + +Starting with Bazel 1.0, you can build any rule without Bash unless it is a: + +- `genrule`, because genrules execute Bash commands +- `sh_binary` or `sh_test` rule, because these inherently need Bash +- Starlark rule that uses `ctx.actions.run_shell()` or `ctx.resolve_command()` + +However, `genrule` is often used for simple tasks like +[copying a file](https://github.com/bazelbuild/bazel-skylib/blob/main/rules/copy_file.bzl) +or [writing a text file](https://github.com/bazelbuild/bazel-skylib/blob/main/rules/write_file.bzl). +Instead of using `genrule` (and depending on Bash) you may find a suitable rule +in the +[bazel-skylib repository](https://github.com/bazelbuild/bazel-skylib/tree/main/rules). +When built on Windows, **these rules do not require Bash**. + +#### Using bazel test without Bash + +Bazel versions before 1.0 used to require Bash to `bazel test` anything. + +Starting with Bazel 1.0, you can test any rule without Bash, except when: + +- you use `--run_under` +- the test rule itself requires Bash (because its executable is a shell script) + +#### Using bazel run without Bash + +Bazel versions before 1.0 used to require Bash to `bazel run` anything. + +Starting with Bazel 1.0, you can run any rule without Bash, except when: + +- you use `--run_under` or `--script_path` +- the test rule itself requires Bash (because its executable is a shell script) + +#### Using sh\_binary and sh\_* rules, and ctx.actions.run_shell() without Bash + +You need Bash to build and test `sh_*` rules, and to build and test Starlark +rules that use `ctx.actions.run_shell()` and `ctx.resolve_command()`. This +applies not only to rules in your project, but to rules in any of the external +repositories your project depends on (even transitively). + +In the future, there may be an option to use Windows Subsystem for +Linux (WSL) to build these rules, but currently it is not a priority for +the Bazel-on-Windows subteam. + +### Setting environment variables + +Environment variables you set in the Windows Command Prompt (`cmd.exe`) are only +set in that command prompt session. If you start a new `cmd.exe`, you need to +set the variables again. To always set the variables when `cmd.exe` starts, you +can add them to the User variables or System variables in the `Control Panel > +System Properties > Advanced > Environment Variables...` dialog box. + +## Build on Windows + +### Build C++ with MSVC + +To build C++ targets with MSVC, you need: + +* [The Visual C++ compiler](/install/windows#install-vc). + +* (Optional) The `BAZEL_VC` and `BAZEL_VC_FULL_VERSION` environment variable. + + Bazel automatically detects the Visual C++ compiler on your system. + To tell Bazel to use a specific VC installation, you can set the + following environment variables: + + For Visual Studio 2017 and 2019, set one of `BAZEL_VC`. Additionally you may also set `BAZEL_VC_FULL_VERSION`. + + * `BAZEL_VC` the Visual C++ Build Tools installation directory + + ``` + set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC + ``` + + * `BAZEL_VC_FULL_VERSION` (Optional) Only for Visual Studio 2017 and 2019, the full version + number of your Visual C++ Build Tools. You can choose the exact Visual C++ Build Tools + version via `BAZEL_VC_FULL_VERSION` if more than one version are installed, otherwise Bazel + will choose the latest version. + + ``` + set BAZEL_VC_FULL_VERSION=14.16.27023 + ``` + + For Visual Studio 2015 or older, set `BAZEL_VC`. (`BAZEL_VC_FULL_VERSION` is not supported.) + + * `BAZEL_VC` the Visual C++ Build Tools installation directory + + ``` + set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC + ``` + +* The [Windows + SDK](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk). + + The Windows SDK contains header files and libraries you need when building + Windows applications, including Bazel itself. By default, the latest Windows SDK installed will + be used. You also can specify Windows SDK version by setting `BAZEL_WINSDK_FULL_VERSION`. You + can use a full Windows 10 SDK number such as 10.0.10240.0, or specify 8.1 to use the Windows 8.1 + SDK (only one version of Windows 8.1 SDK is available). Please make sure you have the specified + Windows SDK installed. + + **Requirement**: This is supported with VC 2017 and 2019. The standalone VC 2015 Build Tools doesn't + support selecting Windows SDK, you'll need the full Visual Studio 2015 installation, otherwise + `BAZEL_WINSDK_FULL_VERSION` will be ignored. + + ``` + set BAZEL_WINSDK_FULL_VERSION=10.0.10240.0 + ``` + +If everything is set up, you can build a C++ target now! + +Try building a target from one of our [sample +projects](https://github.com/bazelbuild/bazel/tree/master/examples): + +``` +bazel build //examples/cpp:hello-world +bazel-bin\examples\cpp\hello-world.exe +``` + +By default, the built binaries target x64 architecture. To build for ARM64 +architecture, use + +```none +--platforms=//:windows_arm64 --extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows +``` + +You can introduce `@local_config_cc` in `MODULE.bazel` with + +```python +bazel_dep(name = "rules_cc", version = "0.1.1") +cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension") +use_repo(cc_configure, "local_config_cc") +``` + +To build and use Dynamically Linked Libraries (DLL files), see [this +example](https://github.com/bazelbuild/bazel/tree/master/examples/windows/dll). + +**Command Line Length Limit**: To prevent the +[Windows command line length limit issue](https://github.com/bazelbuild/bazel/issues/5163), +enable the compiler parameter file feature via `--features=compiler_param_file`. + +### Build C++ with Clang + +From 0.29.0, Bazel supports building with LLVM's MSVC-compatible compiler driver (`clang-cl.exe`). + +**Requirement**: To build with Clang, you have to install **both** +[LLVM](http://releases.llvm.org/download.html) and Visual C++ Build tools, +because although you use `clang-cl.exe` as compiler, you still need to link to +Visual C++ libraries. + +Bazel can automatically detect LLVM installation on your system, or you can explicitly tell +Bazel where LLVM is installed by `BAZEL_LLVM`. + +* `BAZEL_LLVM` the LLVM installation directory + + ```posix-terminal + set BAZEL_LLVM=C:\Program Files\LLVM + ``` + +To enable the Clang toolchain for building C++, there are several situations. + +* In Bazel 7.0.0 and newer: Add a platform target to your `BUILD file` (eg. the + top level `BUILD` file): + + ``` + platform( + name = "x64_windows-clang-cl", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@bazel_tools//tools/cpp:clang-cl", + ], + ) + ``` + + Then enable the Clang toolchain by specifying the following build flags: + + ``` + --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl --extra_execution_platforms=//:x64_windows-clang-cl + ``` + +* In Bazel older than 7.0.0 but newer than 0.28: Enable the Clang toolchain by + a build flag `--compiler=clang-cl`. + + If your build sets the flag + [\-\-incompatible_enable_cc_toolchain_resolution] + (https://github.com/bazelbuild/bazel/issues/7260) + to `true`, then use the approach for Bazel 7.0.0. + +* In Bazel 0.28 and older: Clang is not supported. + +### Build Java + +To build Java targets, you need: + +* [The Java SE Development Kit](/install/windows#install-jdk) + +On Windows, Bazel builds two output files for `java_binary` rules: + +* a `.jar` file +* a `.exe` file that can set up the environment for the JVM and run the binary + +Try building a target from one of our [sample +projects](https://github.com/bazelbuild/bazel/tree/master/examples): + +``` + bazel build //examples/java-native/src/main/java/com/example/myproject:hello-world + bazel-bin\examples\java-native\src\main\java\com\example\myproject\hello-world.exe +``` + +### Build Python + +To build Python targets, you need: + +* The [Python interpreter](/install/windows#install-python) + +On Windows, Bazel builds two output files for `py_binary` rules: + +* a self-extracting zip file +* an executable file that can launch the Python interpreter with the + self-extracting zip file as the argument + +You can either run the executable file (it has a `.exe` extension) or you can run +Python with the self-extracting zip file as the argument. + +Try building a target from one of our [sample +projects](https://github.com/bazelbuild/bazel/tree/master/examples): + +``` + bazel build //examples/py_native:bin + bazel-bin\examples\py_native\bin.exe + python bazel-bin\examples\py_native\bin.zip +``` + +If you are interested in details about how Bazel builds Python targets on +Windows, check out this [design +doc](https://github.com/bazelbuild/bazel-website/blob/master/designs/_posts/2016-09-05-build-python-on-windows.md). diff --git a/contribute/docs-style-guide.mdx b/contribute/docs-style-guide.mdx deleted file mode 100644 index f50c9eb..0000000 --- a/contribute/docs-style-guide.mdx +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: 'Bazel docs style guide' ---- - - - -Thank you for contributing to Bazel's documentation. This serves as a quick -documentation style guide to get you started. For any style questions not -answered by this guide, follow the -[Google developer documentation style guide](https://developers.google.com/style). - -## Defining principles - -Bazel docs should uphold these principles: - -- **Concise.** Use as few words as possible. -- **Clear.** Use plain language. Write without jargon for a fifth-grade - reading level. -- **Consistent.** Use the same words or phrases for repeated concepts - throughout the docs. -- **Correct.** Write in a way where the content stays correct for as long as - possible by avoiding time-based information and promises for the future. - -## Writing - -This section contains basic writing tips. - -### Headings - -- Page-level headings start at H2. (H1 headings are used as page titles.) -- Make headers as short as is sensible. This way, they fit in the TOC - without wrapping. - - - Yes: Permissions - - No: A brief note on permissions - -- Use sentence case for headings - - - Yes: Set up your workspace - - No: Set Up Your Workspace - -- Try to make headings task-based or actionable. If headings are conceptual, - it may be based around understanding, but write to what the user does. - - - Yes: Preserving graph order - - No: On the preservation of graph order - -### Names - -- Capitalize proper nouns, such as Bazel and Starlark. - - - Yes: At the end of the build, Bazel prints the requested targets. - - No: At the end of the build, bazel prints the requested targets. - -- Keep it consistent. Don't introduce new names for existing concepts. Where - applicable, use the term defined in the - [Glossary](/reference/glossary). - - - For example, if you're writing about issuing commands on a - terminal, don't use both terminal and command line on the page. - -### Page scope - -- Each page should have one purpose and that should be defined at the - beginning. This helps readers find what they need quicker. - - - Yes: This page covers how to install Bazel on Windows. - - No: (No introductory sentence.) - -- At the end of the page, tell the reader what to do next. For pages where - there is no clear action, you can include links to similar concepts, - examples, or other avenues for exploration. - -### Subject - -In Bazel documentation, the audience should primarily be users—the people using -Bazel to build their software. - -- Address your reader as "you". (If for some reason you can't use "you", - use gender-neutral language, such as they.) - - Yes: To build Java code using Bazel, - you must install a JDK. - - **MAYBE:** For users to build Java code with Bazel, they must install a JDK. - - No: For a user to build Java code with - Bazel, he or she must install a JDK. - -- If your audience is NOT general Bazel users, define the audience at the - beginning of the page or in the section. Other audiences can include - maintainers, contributors, migrators, or other roles. -- Avoid "we". In user docs, there is no author; just tell people what's - possible. - - Yes: As Bazel evolves, you should update your code base to maintain - compatibility. - - No: Bazel is evolving, and we will make changes to Bazel that at - times will be incompatible and require some changes from Bazel users. - -### Temporal - -Where possible, avoid terms that orient things in time, such as referencing -specific dates (Q2 2022) or saying "now", "currently", or "soon." These go -stale quickly and could be incorrect if it's a future projection. Instead, -specify a version level instead, such as "Bazel X.x and higher supports -\ or a GitHub issue link. - -- Yes: Bazel 0.10.0 or later supports - remote caching. -- No: Bazel will soon support remote - caching, likely in October 2017. - -### Tense - -- Use present tense. Avoid past or future tense unless absolutely necessary - for clarity. - - Yes: Bazel issues an error when it - finds dependencies that don't conform to this rule. - - No: If Bazel finds a dependency that - does not conform to this rule, Bazel will issue an error. - -- Where possible, use active voice (where a subject acts upon an object) not - passive voice (where an object is acted upon by a subject). Generally, - active voice makes sentences clearer because it shows who is responsible. If - using active voice detracts from clarity, use passive voice. - - Yes: Bazel initiates X and uses the - output to build Y. - - No: X is initiated by Bazel and then - afterward Y will be built with the output. - -### Tone - -Write with a business friendly tone. - -- Avoid colloquial language. It's harder to translate phrases that are - specific to English. - - Yes: Good rulesets - - No: So what is a good ruleset? - -- Avoid overly formal language. Write as though you're explaining the - concept to someone who is curious about tech, but doesn't know the details. - -## Formatting - -### File type - -For readability, wrap lines at 80 characters. Long links or code snippets -may be longer, but should start on a new line. For example: - -Note: Where possible, use Markdown instead of HTML in your files. Follow the -[GitHub Markdown Syntax Guide](https://guides.github.com/features/mastering-markdown/#syntax) -for recommended Markdown style. - -### Links - -- Use descriptive link text instead of "here" or "below". This practice - makes it easier to scan a doc and is better for screen readers. - - Yes: For more details, see [Installing Bazel]. - - No: For more details, see [here]. - -- End the sentence with the link, if possible. - - Yes: For more details, see [link]. - - No: See [link] for more information. - -### Lists - -- Use an ordered list to describe how to accomplish a task with steps -- Use an unordered list to list things that aren't task based. (There should - still be an order of sorts, such as alphabetical, importance, etc.) -- Write with parallel structure. For example: - 1. Make all the list items sentences. - 1. Start with verbs that are the same tense. - 1. Use an ordered list if there are steps to follow. - -### Placeholders - -- Use angle brackets to denote a variable that users should change. - In Markdown, escape the angle brackets with a back slash: `\`. - - Yes: `bazel help `: Prints - help and options for `` - - No: bazel help _command_: Prints help - and options for "command" - -- Especially for complicated code samples, use placeholders that make sense - in context. - -### Table of contents - -Use the auto-generated TOC supported by the site. Don't add a manual TOC. - -## Code - -Code samples are developers' best friends. You probably know how to write these -already, but here are a few tips. - -If you're referencing a small snippet of code, you can embed it in a sentence. -If you want the reader to use the code, such as copying a command, use a code -block. - -### Code blocks - -- Keep it short. Eliminate all redundant or unnecessary text from a code - sample. -- In Markdown, specify the type of code block by adding the sample's language. - -``` -```shell -... -``` - -- Separate commands and output into different code blocks. - -### Inline code formatting - -- Use code style for filenames, directories, paths, and small bits of code. -- Use inline code styling instead of _italics_, "quotes," or **bolding**. - - Yes: `bazel help `: Prints - help and options for `` - - No: bazel help _command_: Prints help - and options for "command" diff --git a/copy-upstream-docs.sh b/copy-upstream-docs.sh index e45aa66..ea38846 100755 --- a/copy-upstream-docs.sh +++ b/copy-upstream-docs.sh @@ -24,44 +24,35 @@ LOCAL_FILES=" # This output pasted from a CI job - we should burn it down to zero BROKEN_FILES=" community/roadmaps-configurability.mdx -community/users.mdx concepts/build-files.mdx concepts/dependencies.mdx concepts/labels.mdx configure/integrate-cpp.mdx -configure/windows.mdx +contribute/docs-style-guide.mdx contribute/search.mdx docs/cc-toolchain-config-reference.mdx -docs/mobile-install.mdx docs/user-manual.mdx extending/config.mdx -extending/legacy-macros.mdx -extending/macros.mdx -external/extension.mdx -external/faq.mdx -external/migration.mdx external/mod-command.mdx -external/overview.mdx external/registry.mdx -external/vendor.mdx -install/windows.mdx -query/guide.mdx +external/migration_tool.mdx query/language.mdx query/quickstart.mdx -reference/flag-cheatsheet.mdx -reference/test-encyclopedia.mdx -reference/command-line-reference.mdx -reference/be/be-nav.mdx reference/be/functions.mdx reference/be/platforms-and-toolchains.mdx -release/rolling.mdx -remote/ci.mdx +reference/command-line-reference.mdx +reference/flag-cheatsheet.mdx +reference/test-encyclopedia.mdx remote/dynamic.mdx -rules/language.mdx +rules/lib/globals/bzl.mdx +rules/lib/repo/cache.mdx +rules/lib/repo/git.mdx +rules/lib/repo/http.mdx +rules/lib/repo/local.mdx +rules/lib/repo/utils.mdx +rules/lib/globals/module.mdx rules/windows.mdx run/build.mdx -start/go.mdx -tutorials/ccp-toolchain-config.mdx " # Verify that at least one source exists diff --git a/docs.json b/docs.json index b79de5b..b54c70a 100644 --- a/docs.json +++ b/docs.json @@ -166,17 +166,22 @@ { "group": "Build encyclopedia", "pages": [ - "reference/be/c-cpp", + "reference/be/overview", "reference/be/common-definitions", - "reference/be/extra-actions", - "reference/be/general", - "reference/be/java", "reference/be/make-variables", - "reference/be/objective-c", - "reference/be/overview", - "reference/be/protocol-buffer", - "reference/be/python", - "reference/be/shell" + "reference/be/functions" + ] + }, + { + "group": "Command line reference", + "pages": [ + "reference/command-line-reference" + ] + }, + { + "group": "Query Language", + "pages": [ + "query/language" ] }, { @@ -186,9 +191,9 @@ ] }, { - "group": "Skyframe", + "group": "Flag cheatsheet", "pages": [ - "reference/skyframe" + "reference/flag-cheatsheet" ] } ] @@ -254,6 +259,7 @@ { "group": "Programs", "pages": [ + "community/update", "community/sig", "community/experts", "community/partners", @@ -524,6 +530,7 @@ { "group": "Programs", "pages": [ + "8.4.2/community/update", "8.4.2/community/sig", "8.4.2/community/experts", "8.4.2/community/partners", @@ -794,6 +801,7 @@ { "group": "Programs", "pages": [ + "8.3.1/community/update", "8.3.1/community/sig", "8.3.1/community/experts", "8.3.1/community/partners", @@ -1064,6 +1072,7 @@ { "group": "Programs", "pages": [ + "8.2.1/community/update", "8.2.1/community/sig", "8.2.1/community/experts", "8.2.1/community/partners", @@ -1334,6 +1343,7 @@ { "group": "Programs", "pages": [ + "8.1.1/community/update", "8.1.1/community/sig", "8.1.1/community/experts", "8.1.1/community/partners", @@ -1604,6 +1614,7 @@ { "group": "Programs", "pages": [ + "8.0.1/community/update", "8.0.1/community/sig", "8.0.1/community/experts", "8.0.1/community/partners", @@ -1874,6 +1885,7 @@ { "group": "Programs", "pages": [ + "7.6.2/community/update", "7.6.2/community/sig", "7.6.2/community/experts", "7.6.2/community/partners", @@ -2144,6 +2156,7 @@ { "group": "Programs", "pages": [ + "6.5.0/community/update", "6.5.0/community/sig", "6.5.0/community/experts", "6.5.0/community/partners", @@ -2414,6 +2427,7 @@ { "group": "Programs", "pages": [ + "5.4.1/community/update", "5.4.1/community/sig", "5.4.1/community/experts", "5.4.1/community/partners", diff --git a/docs/android-instrumentation-test.mdx b/docs/android-instrumentation-test.mdx index bf0ff76..fca7b57 100644 --- a/docs/android-instrumentation-test.mdx +++ b/docs/android-instrumentation-test.mdx @@ -309,7 +309,7 @@ Bazel currently supports x86-based emulators only. For better performance, use ## Running tests To run tests, add these lines to your project's -`{{ '' }}project root{{ '' }}:{{ '' }}/.bazelrc` file. +`project root:/.bazelrc` file. ``` # Configurations for testing with Bazel @@ -394,7 +394,7 @@ junit:junit ``` One way to organize these dependencies is to create a `//:test_deps` shared -library in your `{{ "" }}project root{{ "" }}/BUILD.bazel` file: +library in your `project root/BUILD.bazel` file: ```python java_library( @@ -411,7 +411,7 @@ java_library( ) ``` -Then, add the required dependencies in `{{ "" }}project root{{ "" }}/WORKSPACE`: +Then, add the required dependencies in `project root/WORKSPACE`: ```python load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") diff --git a/docs/android-ndk.mdx b/docs/android-ndk.mdx index 8c65f86..b434797 100644 --- a/docs/android-ndk.mdx +++ b/docs/android-ndk.mdx @@ -318,7 +318,7 @@ use [Bazel wild-cards](/run/build#specifying-build-targets) like `/...` and `:all`. These flags can be put into a `bazelrc` config (one for each ABI), in -`{{ "" }}project{{ "" }}/.bazelrc`: +`project/.bazelrc`: ``` common:android_x86 --platforms=//my/platforms:x86 diff --git a/docs/bazel-and-cpp.mdx b/docs/bazel-and-cpp.mdx index 9ade384..73e502c 100644 --- a/docs/bazel-and-cpp.mdx +++ b/docs/bazel-and-cpp.mdx @@ -71,7 +71,7 @@ Follow these guidelines for include paths: * Make all include paths relative to the workspace directory. * Use quoted includes (`#include "foo/bar/baz.h"`) for non-system headers, not - angle-brackets (`#include `). + angle-brackets (`#include <foo/bar/baz.h>`). * Avoid using UNIX directory shortcuts, such as `.` (current directory) or `..` (parent directory). diff --git a/docs/mobile-install.mdx b/docs/mobile-install.mdx new file mode 100644 index 0000000..8bc4a67 --- /dev/null +++ b/docs/mobile-install.mdx @@ -0,0 +1,204 @@ +--- +title: 'bazel mobile-install' +--- + + + + +

Fast iterative development for Android

+ +This page describes how `bazel mobile-install` makes iterative development +for Android much faster. It describes the benefits of this approach versus the +drawbacks of separate build and install steps. + +## Summary + +To install small changes to an Android app very quickly, do the following: + + 1. Find the `android_binary` rule of the app you want to install. + 2. Connect your device to `adb`. + 3. Run `bazel mobile-install :your_target`. App startup will be a little + slower than usual. + 4. Edit the code or Android resources. + 5. Run `bazel mobile-install :your_target`. + 6. Enjoy a fast and minimal incremental installation! + +Some command line options to Bazel that may be useful: + + - `--adb` tells Bazel which adb binary to use + - `--adb_arg` can be used to add extra arguments to the command line of `adb`. + One useful application of this is to select which device you want to install + to if you have multiple devices connected to your workstation: + `bazel mobile-install :your_target -- --adb_arg=-s --adb_arg=` + +When in doubt, look at the +[example](https://github.com/bazelbuild/rules_android/tree/main/examples/basicapp), +contact us on [Google Groups](https://groups.google.com/forum/#!forum/bazel-discuss), +or [file a GitHub issue](https://github.com/bazelbuild/rules_android/issues) + +## Introduction + +One of the most important attributes of a developer's toolchain is speed: there +is a world of difference between changing the code and seeing it run within a +second and having to wait minutes, sometimes hours, before you get any feedback +on whether your changes do what you expect them to. + +Unfortunately, the traditional Android toolchain for building an .apk entails +many monolithic, sequential steps and all of these have to be done in order to +build an Android app. At Google, waiting five minutes to build a single-line +change was not unusual on larger projects like Google Maps. + +`bazel mobile-install` makes iterative development for Android much faster by +using a combination of change pruning, work sharding, and clever manipulation of +Android internals, all without changing any of your app's code. + +## Problems with traditional app installation + +Building an Android app has some issues, including: + +- Dexing. By default, the Dexer tool (historically `dx`, now `d8` or `r8`) +is invoked exactly once in the build and it does not know how to reuse work from +previous builds: it dexes every method again, even though only one method was +changed. + +- Uploading data to the device. adb does not use the full bandwidth of a USB 2.0 +connection, and larger apps can take a lot of time to upload. The entire app is +uploaded, even if only small parts have changed, for example, a resource or a +single method, so this can be a major bottleneck. + +- Compilation to native code. Android L introduced ART, a new Android runtime, +which compiles apps ahead-of-time rather than compiling them just-in-time like +Dalvik. This makes apps much faster at the cost of longer installation +time. This is a good tradeoff for users because they typically install an app +once and use it many times, but results in slower development where an app is +installed many times and each version is run at most a handful of times. + +## The approach of `bazel mobile-install` + +`bazel mobile-install `makes the following improvements: + + - Sharded desugaring and dexing. After building the app's Java code, Bazel + shards the class files into approximately equal-sized parts and invokes `d8` + separately on them. `d8` is not invoked on shards that did not change since + the last build. These shards are then compiled into separate sharded APKs. + + - Incremental file transfer. Android resources, .dex files, and native + libraries are removed from the main .apk and are stored in under a separate + mobile-install directory. This makes it possible to update code and Android + resources independently without reinstalling the whole app. Thus, + transferring the files takes less time and only the .dex files that have + changed are recompiled on-device. + + - Sharded installation. Mobile-install uses Android Studio's + [`apkdeployer`](https://maven.google.com/web/index.html?q=deployer#com.android.tools.apkdeployer:apkdeployer) + tool to combine sharded APKs on the connected device and provide a cohesive + experience. + +### Sharded Dexing + +Sharded dexing is reasonably straightforward: once the .jar files are built, a +[tool](https://github.com/bazelbuild/rules_android/blob/main/src/tools/java/com/google/devtools/build/android/ziputils/DexMapper.java) +shards them into separate .jar files of approximately equal size, then invokes +`d8` on those that were changed since the previous build. The logic that +determines which shards to dex is not specific to Android: it just uses the +general change pruning algorithm of Bazel. + +The first version of the sharding algorithm simply ordered the .class files +alphabetically, then cut the list up into equal-sized parts, but this proved to +be suboptimal: if a class was added or removed (even a nested or an anonymous +one), it would cause all the classes alphabetically after it to shift by one, +resulting in dexing those shards again. Thus, it was decided to shard Java +packages rather than individual classes. Of course, this still results in +dexing many shards if a new package is added or removed, but that is much less +frequent than adding or removing a single class. + +The number of shards is controlled by command-line configuration, using the +`--define=num_dex_shards=N` flag. In an ideal world, Bazel would +automatically determine how many shards are best, but Bazel currently must know +the set of actions (for example, commands to be executed during the build) before +executing any of them, so it cannot determine the optimal number of shards +because it doesn't know how many Java classes there will eventually be in the +app. Generally speaking, the more shards, the faster the build and the +installation will be, but the slower app startup becomes, because the dynamic +linker has to do more work. The sweet spot is usually between 10 and 50 shards. + +### Incremental deployment + +Incremental APK shard transfer and installation is now handled by the +`apkdeployer` utility described in ["The approach of mobile-install"](#approach-mobile-install). +Whereas earlier (native) versions of mobile-install required manually tracking +first-time installations and selectively apply the `--incremental` +flag on subsequent installation, the most recent version in [`rules_android`](https://github.com/bazelbuild/rules_android/tree/main/mobile_install) +has been greatly simplified. The same mobile-install +invocation can be used regardless of how many times the app has been installed +or reinstalled. + +At a high level, the `apkdeployer` tool is a wrapper around various `adb` +sub-commands. The main entrypoint logic can be found in the +[`com.android.tools.deployer.Deployer`](https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:deploy/deployer/src/main/java/com/android/tools/deployer/Deployer.java) +class, with other utility classes colocated in the same package. +The `Deployer` class ingests, among other things, a list of paths to split +APKs and a protobuf with information about the installation, and leverages +deployment features for [Android app bundles](https://developer.android.com/guide/app-bundle) +in order to create an install session and incrementally deploy app splits. +See the [`ApkPreInstaller`](https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:deploy/deployer/src/main/java/com/android/tools/deployer/ApkPreInstaller.java) +and [`ApkInstaller`](https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:deploy/deployer/src/main/java/com/android/tools/deployer/ApkInstaller.java) +classes for implementation details. + +## Results + +### Performance + +In general, `bazel mobile-install` results in a 4x to 10x speedup of building +and installing large apps after a small change. + +The following numbers were computed for a few Google products: + + + +This, of course, depends on the nature of the change: recompilation after +changing a base library takes more time. + +### Limitations + +The tricks the stub application plays don't work in every case. +The following cases highlight where it does not work as expected: + + - Mobile-install is only supported via the Starlark rules of `rules_android`. + See the ["brief history of mobile-install"](#mobile-install-history) for + more detail. + + - Only devices running ART are supported. Mobile-install uses API and runtime features + that only exist on devices running ART, not Dalvik. Any Android runtime more + recent than Android L (API 21+) should be compatible. + + - Bazel itself must be run with a tool Java runtime _and_ language version + of 17 or higher. + + - Bazel versions prior to 8.4.0 must specify some additional flags for + mobile-install. See [the Bazel Android tutorial](/start/android-app). These + flags inform Bazel where the Starlark mobile-install aspect is and which + rules are supported. + +### A brief history of mobile-install +Earlier Bazel versions _natively_ included built-in build and test rules for +popular languages and ecosystems such as C++, Java, and Android. These rules +were therefore referred to as _native_ rules. Bazel 8 (released in 2024) removed +support for these rules because many of them had been migrated to the +[Starlark](/rules/language) language. See the ["Bazel 8.0 LTS blog post"](https://blog.bazel.build/2024/12/09/bazel-8-release.html) +for more details. + +The legacy native Android rules also supported a legacy _native_ version of +mobile-install functionality. This is referred to as "mobile-install v1" or +"native mobile-install" now. This functionality was deleted in Bazel 8, along +with the built-in Android rules. + +Now, all mobile-install functionality, as well as all Android build and test +rules, are implemented in Starlark and reside in the `rules_android` GitHub +repository. The latest version is known as "mobile-install v3" or "MIv3". + +_Naming note_: There was a "mobile-install **v2**" available only internally +at Google at one point, but this was never published externally, and only v3 +continues to be used for both Google-internal and OSS rules_android deployment. + + diff --git a/extending/concepts.mdx b/extending/concepts.mdx index 893e9cc..e634c61 100644 --- a/extending/concepts.mdx +++ b/extending/concepts.mdx @@ -4,6 +4,7 @@ title: 'Extension Overview' +{/* [TOC] */} This page describes how to extend the BUILD language using macros and rules. diff --git a/extending/legacy-macros.mdx b/extending/legacy-macros.mdx new file mode 100644 index 0000000..9c2b8e2 --- /dev/null +++ b/extending/legacy-macros.mdx @@ -0,0 +1,254 @@ +--- +title: 'Legacy Macros' +--- + + + +Legacy macros are unstructured functions called from `BUILD` files that can +create targets. By the end of the +[loading phase](/extending/concepts#evaluation-model), legacy macros don't exist +anymore, and Bazel sees only the concrete set of instantiated rules. + +## Why you shouldn't use legacy macros (and should use Symbolic macros instead) + +Where possible you should use [symbolic macros](macros.md#macros). + +Symbolic macros + +* Prevent action at a distance +* Make it possible to hide implementation details through granular visibility +* Take typed attributes, which in turn means automatic label and select + conversion. +* Are more readable +* Will soon have [lazy evaluation](macros.md#laziness) + +## Usage + +The typical use case for a macro is when you want to reuse a rule. + +For example, genrule in a `BUILD` file generates a file using `//:generator` +with a `some_arg` argument hardcoded in the command: + +```python +genrule( + name = "file", + outs = ["file.txt"], + cmd = "$(location //:generator) some_arg > $@", + tools = ["//:generator"], +) +``` + +Note: `$@` is a +[Make variable](/reference/be/make-variables#predefined_genrule_variables) that +refers to the execution-time locations of the files in the `outs` attribute +list. It is equivalent to `$(locations :file.txt)`. + +If you want to generate more files with different arguments, you may want to +extract this code to a macro function. To create a macro called +`file_generator`, which has `name` and `arg` parameters, we can replace the +genrule with the following: + +```python +load("//path:generator.bzl", "file_generator") + +file_generator( + name = "file", + arg = "some_arg", +) + +file_generator( + name = "file-two", + arg = "some_arg_two", +) + +file_generator( + name = "file-three", + arg = "some_arg_three", +) +``` + +Here, you load the `file_generator` symbol from a `.bzl` file located in the +`//path` package. By putting macro function definitions in a separate `.bzl` +file, you keep your `BUILD` files clean and declarative, The `.bzl` file can be +loaded from any package in the workspace. + +Finally, in `path/generator.bzl`, write the definition of the macro to +encapsulate and parameterize the original genrule definition: + +```python +def file_generator(name, arg, visibility=None): + native.genrule( + name = name, + outs = [name + ".txt"], + cmd = "$(location //:generator) %s > $@" % arg, + tools = ["//:generator"], + visibility = visibility, + ) +``` + +You can also use macros to chain rules together. This example shows chained +genrules, where a genrule uses the outputs of a previous genrule as inputs: + +```python +def chained_genrules(name, visibility=None): + native.genrule( + name = name + "-one", + outs = [name + ".one"], + cmd = "$(location :tool-one) $@", + tools = [":tool-one"], + visibility = ["//visibility:private"], + ) + + native.genrule( + name = name + "-two", + srcs = [name + ".one"], + outs = [name + ".two"], + cmd = "$(location :tool-two) $< $@", + tools = [":tool-two"], + visibility = visibility, + ) +``` + +The example only assigns a visibility value to the second genrule. This allows +macro authors to hide the outputs of intermediate rules from being depended upon +by other targets in the workspace. + +Note: Similar to `$@` for outputs, `$<` expands to the locations of files in the +`srcs` attribute list. + +## Expanding macros + +When you want to investigate what a macro does, use the `query` command with +`--output=build` to see the expanded form: + +```none +$ bazel query --output=build :file +# /absolute/path/test/ext.bzl:42:3 +genrule( + name = "file", + tools = ["//:generator"], + outs = ["//test:file.txt"], + cmd = "$(location //:generator) some_arg > $@", +) +``` + +## Instantiating native rules + +Native rules (rules that don't need a `load()` statement) can be instantiated +from the [native](/rules/lib/toplevel/native) module: + +```python +def my_macro(name, visibility=None): + native.cc_library( + name = name, + srcs = ["main.cc"], + visibility = visibility, + ) +``` + +If you need to know the package name (for example, which `BUILD` file is calling +the macro), use the function +[native.package_name()](/rules/lib/toplevel/native#package_name). Note that +`native` can only be used in `.bzl` files, and not in `BUILD` files. + +## Label resolution in macros + +Since legacy macros are evaluated in the +[loading phase](concepts.md#evaluation-model), label strings such as +`"//foo:bar"` that occur in a legacy macro are interpreted relative to the +`BUILD` file in which the macro is used rather than relative to the `.bzl` file +in which it is defined. This behavior is generally undesirable for macros that +are meant to be used in other repositories, such as because they are part of a +published Starlark ruleset. + +To get the same behavior as for Starlark rules, wrap the label strings with the +[`Label`](/rules/lib/builtins/Label#Label) constructor: + +```python +# @my_ruleset//rules:defs.bzl +def my_cc_wrapper(name, deps = [], **kwargs): + native.cc_library( + name = name, + deps = deps + select({ + # Due to the use of Label, this label is resolved within @my_ruleset, + # regardless of its site of use. + Label("//config:needs_foo"): [ + # Due to the use of Label, this label will resolve to the correct target + # even if the canonical name of @dep_of_my_ruleset should be different + # in the main repo, such as due to repo mappings. + Label("@dep_of_my_ruleset//tools:foo"), + ], + "//conditions:default": [], + }), + **kwargs, + ) +``` + +With the `--incompatible_resolve_select_keys_eagerly` flag enabled, all keys +that are label strings will be automatically resolved to `Label` objects +relative to the package of the file that contains the `select` call. If this is +not chosen, wrap the label string with +[native.package_relative_label()](/rules/lib/toplevel/native#package_relative_label). + +## Debugging + +* `bazel query --output=build //my/path:all` will show you how the `BUILD` + file looks after evaluation. All legacy macros, globs, loops are expanded. + Known limitation: `select` expressions are not shown in the output. + +* You may filter the output based on `generator_function` (which function + generated the rules) or `generator_name` (the name attribute of the macro): + `bash $ bazel query --output=build 'attr(generator_function, my_macro, + //my/path:all)'` + +* To find out where exactly the rule `foo` is generated in a `BUILD` file, you + can try the following trick. Insert this line near the top of the `BUILD` + file: `cc_library(name = "foo")`. Run Bazel. You will get an exception when + the rule `foo` is created (due to a name conflict), which will show you the + full stack trace. + +* You can also use [print](/rules/lib/globals/all#print) for debugging. It + displays the message as a `DEBUG` log line during the loading phase. Except + in rare cases, either remove `print` calls, or make them conditional under a + `debugging` parameter that defaults to `False` before submitting the code to + the depot. + +## Errors + +If you want to throw an error, use the [fail](/rules/lib/globals/all#fail) +function. Explain clearly to the user what went wrong and how to fix their +`BUILD` file. It is not possible to catch an error. + +```python +def my_macro(name, deps, visibility=None): + if len(deps) < 2: + fail("Expected at least two values in deps") + # ... +``` + +## Conventions + +* All public functions (functions that don't start with underscore) that + instantiate rules must have a `name` argument. This argument should not be + optional (don't give a default value). + +* Public functions should use a docstring following + [Python conventions](https://www.python.org/dev/peps/pep-0257/#one-line-docstrings). + +* In `BUILD` files, the `name` argument of the macros must be a keyword + argument (not a positional argument). + +* The `name` attribute of rules generated by a macro should include the name + argument as a prefix. For example, `macro(name = "foo")` can generate a + `cc_library` `foo` and a genrule `foo_gen`. + +* In most cases, optional parameters should have a default value of `None`. + `None` can be passed directly to native rules, which treat it the same as if + you had not passed in any argument. Thus, there is no need to replace it + with `0`, `False`, or `[]` for this purpose. Instead, the macro should defer + to the rules it creates, as their defaults may be complex or may change over + time. Additionally, a parameter that is explicitly set to its default value + looks different than one that is never set (or set to `None`) when accessed + through the query language or build-system internals. + +* Macros should have an optional `visibility` argument. diff --git a/extending/macros.mdx b/extending/macros.mdx new file mode 100644 index 0000000..136665d --- /dev/null +++ b/extending/macros.mdx @@ -0,0 +1,465 @@ +--- +title: 'Macros' +--- + + + +This page covers the basics of using macros and includes typical use cases, +debugging, and conventions. + +A macro is a function called from the `BUILD` file that can instantiate rules. +Macros are mainly used for encapsulation and code reuse of existing rules and +other macros. + +Macros come in two flavors: symbolic macros, which are described on this page, +and [legacy macros](legacy-macros.md). Where possible, we recommend using +symbolic macros for code clarity. + +Symbolic macros offer typed arguments (string to label conversion, relative to +where the macro was called) and the ability to restrict and specify the +visibility of targets created. They are designed to be amenable to lazy +evaluation (which will be added in a future Bazel release). Symbolic macros are +available by default in Bazel 8. Where this document mentions `macros`, it's +referring to **symbolic macros**. + +An executable example of symbolic macros can be found in the +[examples repository](https://github.com/bazelbuild/examples/tree/main/macros). + +## Usage + +Macros are defined in `.bzl` files by calling the +[`macro()`](https://bazel.build/rules/lib/globals/bzl.html#macro) function with +two required parameters: `attrs` and `implementation`. + +### Attributes + +`attrs` accepts a dictionary of attribute name to [attribute +types](https://bazel.build/rules/lib/toplevel/attr#members), which represents +the arguments to the macro. Two common attributes – `name` and `visibility` – +are implicitly added to all macros and are not included in the dictionary passed +to `attrs`. + +```starlark +# macro/macro.bzl +my_macro = macro( + attrs = { + "deps": attr.label_list(mandatory = True, doc = "The dependencies passed to the inner cc_binary and cc_test targets"), + "create_test": attr.bool(default = False, configurable = False, doc = "If true, creates a test target"), + }, + implementation = _my_macro_impl, +) +``` + +Attribute type declarations accept the +[parameters](https://bazel.build/rules/lib/toplevel/attr#parameters), +`mandatory`, `default`, and `doc`. Most attribute types also accept the +`configurable` parameter, which determines whether the attribute accepts +`select`s. If an attribute is `configurable`, it will parse non-`select` values +as an unconfigurable `select` – `"foo"` will become +`select({"//conditions:default": "foo"})`. Learn more in [selects](#selects). + +#### Attribute inheritance + +Macros are often intended to wrap a rule (or another macro), and the macro's +author often wants to forward the bulk of the wrapped symbol's attributes +unchanged, using `**kwargs`, to the macro's main target (or main inner macro). + +To support this pattern, a macro can *inherit attributes* from a rule or another +macro by passing the [rule](https://bazel.build/rules/lib/builtins/rule) or +[macro symbol](https://bazel.build/rules/lib/builtins/macro) to `macro()`'s +`inherit_attrs` argument. (You can also use the special string `"common"` +instead of a rule or macro symbol to inherit the [common attributes defined for +all Starlark build +rules](https://bazel.build/reference/be/common-definitions#common-attributes).) +Only public attributes get inherited, and the attributes in the macro's own +`attrs` dictionary override inherited attributes with the same name. You can +also *remove* inherited attributes by using `None` as a value in the `attrs` +dictionary: + +```starlark +# macro/macro.bzl +my_macro = macro( + inherit_attrs = native.cc_library, + attrs = { + # override native.cc_library's `local_defines` attribute + "local_defines": attr.string_list(default = ["FOO"]), + # do not inherit native.cc_library's `defines` attribute + "defines": None, + }, + ... +) +``` + +The default value of non-mandatory inherited attributes is always overridden to +be `None`, regardless of the original attribute definition's default value. If +you need to examine or modify an inherited non-mandatory attribute – for +example, if you want to add a tag to an inherited `tags` attribute – you must +make sure to handle the `None` case in your macro's implementation function: + +```starlark +# macro/macro.bzl +def _my_macro_impl(name, visibility, tags, **kwargs): + # Append a tag; tags attr is an inherited non-mandatory attribute, and + # therefore is None unless explicitly set by the caller of our macro. + my_tags = (tags or []) + ["another_tag"] + native.cc_library( + ... + tags = my_tags, + **kwargs, + ) + ... +``` + +### Implementation + +`implementation` accepts a function which contains the logic of the macro. +Implementation functions often create targets by calling one or more rules, and +they are usually private (named with a leading underscore). Conventionally, +they are named the same as their macro, but prefixed with `_` and suffixed with +`_impl`. + +Unlike rule implementation functions, which take a single argument (`ctx`) that +contains a reference to the attributes, macro implementation functions accept a +parameter for each argument. + +```starlark +# macro/macro.bzl +def _my_macro_impl(name, visibility, deps, create_test): + cc_library( + name = name + "_cc_lib", + deps = deps, + ) + + if create_test: + cc_test( + name = name + "_test", + srcs = ["my_test.cc"], + deps = deps, + ) +``` + +If a macro inherits attributes, its implementation function *must* have a +`**kwargs` residual keyword parameter, which can be forwarded to the call that +invokes the inherited rule or submacro. (This helps ensure that your macro won't +be broken if the rule or macro which from which you are inheriting adds a new +attribute.) + +### Declaration + +Macros are declared by loading and calling their definition in a `BUILD` file. + +```starlark + +# pkg/BUILD + +my_macro( + name = "macro_instance", + deps = ["src.cc"] + select( + { + "//config_setting:special": ["special_source.cc"], + "//conditions:default": [], + }, + ), + create_tests = True, +) +``` + +This would create targets +`//pkg:macro_instance_cc_lib` and`//pkg:macro_instance_test`. + +Just like in rule calls, if an attribute value in a macro call is set to `None`, +that attribute is treated as if it was omitted by the macro's caller. For +example, the following two macro calls are equivalent: + +```starlark +# pkg/BUILD +my_macro(name = "abc", srcs = ["src.cc"], deps = None) +my_macro(name = "abc", srcs = ["src.cc"]) +``` + +This is generally not useful in `BUILD` files, but is helpful when +programmatically wrapping a macro inside another macro. + +## Details + +### Naming conventions for targets created + +The names of any targets or submacros created by a symbolic macro must +either match the macro's `name` parameter or must be prefixed by `name` followed +by `_` (preferred), `.` or `-`. For example, `my_macro(name = "foo")` may only +create files or targets named `foo`, or prefixed by `foo_`, `foo-` or `foo.`, +for example, `foo_bar`. + +Targets or files that violate macro naming convention can be declared, but +cannot be built and cannot be used as dependencies. + +Non-macro files and targets within the same package as a macro instance should +*not* have names that conflict with potential macro target names, though this +exclusivity is not enforced. We are in the progress of implementing +[lazy evaluation](#laziness) as a performance improvement for Symbolic macros, +which will be impaired in packages that violate the naming schema. + +### Restrictions + +Symbolic macros have some additional restrictions compared to legacy macros. + +Symbolic macros + +* must take a `name` argument and a `visibility` argument +* must have an `implementation` function +* may not return values +* may not mutate their arguments +* may not call `native.existing_rules()` unless they are special `finalizer` + macros +* may not call `native.package()` +* may not call `glob()` +* may not call `native.environment_group()` +* must create targets whose names adhere to the [naming schema](#naming) +* can't refer to input files that weren't declared or passed in as an argument +* can't refer to private targets of their callers (see + [visibility and macros](#visibility) for more details). + +### Visibility and macros + +The [visibility](/concepts/visibility) system helps protect the implementation +details of both (symbolic) macros and their callers. + +By default, targets created in a symbolic macro are visible within the macro +itself, but not necessarily to the macro's caller. The macro can "export" a +target as a public API by forwarding the value of its own `visibility` +attribute, as in `some_rule(..., visibility = visibility)`. + +The key ideas of macro visibility are: + +1. Visibility is checked based on what macro declared the target, not what + package called the macro. + + * In other words, being in the same package does not by itself make one + target visible to another. This protects the macro's internal targets + from becoming dependencies of other macros or top-level targets in the + package. + +1. All `visibility` attributes, on both rules and macros, automatically + include the place where the rule or macro was called. + + * Thus, a target is unconditionally visible to other targets declared in the + same macro (or the `BUILD` file, if not in a macro). + +In practice, this means that when a macro declares a target without setting its +`visibility`, the target defaults to being internal to the macro. (The package's +[default visibility](/reference/be/functions#package.default_visibility) does +not apply within a macro.) Exporting the target means that the target is visible +to whatever the macro's caller specified in the macro's `visibility` attribute, +plus the package of the macro's caller itself, as well as the macro's own code. +Another way of thinking of it is that the visibility of a macro determines who +(aside from the macro itself) can see the macro's exported targets. + +```starlark +# tool/BUILD +... +some_rule( + name = "some_tool", + visibility = ["//macro:__pkg__"], +) +``` + +```starlark +# macro/macro.bzl + +def _impl(name, visibility): + cc_library( + name = name + "_helper", + ... + # No visibility passed in. Same as passing `visibility = None` or + # `visibility = ["//visibility:private"]`. Visible to the //macro + # package only. + ) + cc_binary( + name = name + "_exported", + deps = [ + # Allowed because we're also in //macro. (Targets in any other + # instance of this macro, or any other macro in //macro, can see it + # too.) + name + "_helper", + # Allowed by some_tool's visibility, regardless of what BUILD file + # we're called from. + "//tool:some_tool", + ], + ... + visibility = visibility, + ) + +my_macro = macro(implementation = _impl, ...) +``` + +```starlark +# pkg/BUILD +load("//macro:macro.bzl", "my_macro") +... + +my_macro( + name = "foo", + ... +) + +some_rule( + ... + deps = [ + # Allowed, its visibility is ["//pkg:__pkg__", "//macro:__pkg__"]. + ":foo_exported", + # Disallowed, its visibility is ["//macro:__pkg__"] and + # we are not in //macro. + ":foo_helper", + ] +) +``` + +If `my_macro` were called with `visibility = ["//other_pkg:__pkg__"]`, or if +the `//pkg` package had set its `default_visibility` to that value, then +`//pkg:foo_exported` could also be used within `//other_pkg/BUILD` or within a +macro defined in `//other_pkg:defs.bzl`, but `//pkg:foo_helper` would remain +protected. + +A macro can declare that a target is visible to a friend package by passing +`visibility = ["//some_friend:__pkg__"]` (for an internal target) or +`visibility = visibility + ["//some_friend:__pkg__"]` (for an exported one). +Note that it is an antipattern for a macro to declare a target with public +visibility (`visibility = ["//visibility:public"]`). This is because it makes +the target unconditionally visible to every package, even if the caller +specified a more restricted visibility. + +All visibility checking is done with respect to the innermost currently running +symbolic macro. However, there is a visibility delegation mechanism: If a macro +passes a label as an attribute value to an inner macro, any usages of the label +in the inner macro are checked with respect to the outer macro. See the +[visibility page](/concepts/visibility#symbolic-macros) for more details. + +Remember that legacy macros are entirely transparent to the visibility system, +and behave as though their location is whatever BUILD file or symbolic macro +they were called from. + +#### Finalizers and visibility + +Targets declared in a rule finalizer, in addition to seeing targets following +the usual symbolic macro visibility rules, can *also* see all targets which are +visible to the finalizer target's package. + +This means that if you migrate a `native.existing_rules()`-based legacy macro to +a finalizer, the targets declared by the finalizer will still be able to see +their old dependencies. + +However, note that it's possible to declare a target in a symbolic macro such +that a finalizer's targets cannot see it under the visibility system – even +though the finalizer can *introspect* its attributes using +`native.existing_rules()`. + +### Selects + +If an attribute is `configurable` (the default) and its value is not `None`, +then the macro implementation function will see the attribute value as wrapped +in a trivial `select`. This makes it easier for the macro author to catch bugs +where they did not anticipate that the attribute value could be a `select`. + +For example, consider the following macro: + +```starlark +my_macro = macro( + attrs = {"deps": attr.label_list()}, # configurable unless specified otherwise + implementation = _my_macro_impl, +) +``` + +If `my_macro` is invoked with `deps = ["//a"]`, that will cause `_my_macro_impl` +to be invoked with its `deps` parameter set to `select({"//conditions:default": +["//a"]})`. If this causes the implementation function to fail (say, because the +code tried to index into the value as in `deps[0]`, which is not allowed for +`select`s), the macro author can then make a choice: either they can rewrite +their macro to only use operations compatible with `select`, or they can mark +the attribute as nonconfigurable (`attr.label_list(configurable = False)`). The +latter ensures that users are not permitted to pass a `select` value in. + +Rule targets reverse this transformation, and store trivial `select`s as their +unconditional values; in the above example, if `_my_macro_impl` declares a rule +target `my_rule(..., deps = deps)`, that rule target's `deps` will be stored as +`["//a"]`. This ensures that `select`-wrapping does not cause trivial `select` +values to be stored in all targets instantiated by macros. + +If the value of a configurable attribute is `None`, it does not get wrapped in a +`select`. This ensures that tests like `my_attr == None` still work, and that +when the attribute is forwarded to a rule with a computed default, the rule +behaves properly (that is, as if the attribute were not passed in at all). It is +not always possible for an attribute to take on a `None` value, but it can +happen for the `attr.label()` type, and for any inherited non-mandatory +attribute. + +## Finalizers + +A rule finalizer is a special symbolic macro which – regardless of its lexical +position in a BUILD file – is evaluated in the final stage of loading a package, +after all non-finalizer targets have been defined. Unlike ordinary symbolic +macros, a finalizer can call `native.existing_rules()`, where it behaves +slightly differently than in legacy macros: it only returns the set of +non-finalizer rule targets. The finalizer may assert on the state of that set or +define new targets. + +To declare a finalizer, call `macro()` with `finalizer = True`: + +```starlark +def _my_finalizer_impl(name, visibility, tags_filter): + for r in native.existing_rules().values(): + for tag in r.get("tags", []): + if tag in tags_filter: + my_test( + name = name + "_" + r["name"] + "_finalizer_test", + deps = [r["name"]], + data = r["srcs"], + ... + ) + continue + +my_finalizer = macro( + attrs = {"tags_filter": attr.string_list(configurable = False)}, + implementation = _impl, + finalizer = True, +) +``` + +## Laziness + +IMPORTANT: We are in the process of implementing lazy macro expansion and +evaluation. This feature is not available yet. + +Currently, all macros are evaluated as soon as the BUILD file is loaded, which +can negatively impact performance for targets in packages that also have costly +unrelated macros. In the future, non-finalizer symbolic macros will only be +evaluated if they're required for the build. The prefix naming schema helps +Bazel determine which macro to expand given a requested target. + +## Migration troubleshooting + +Here are some common migration headaches and how to fix them. + +* Legacy macro calls `glob()` + +Move the `glob()` call to your BUILD file (or to a legacy macro called from the +BUILD file), and pass the `glob()` value to the symbolic macro using a +label-list attribute: + +```starlark +# BUILD file +my_macro( + ..., + deps = glob(...), +) +``` + +* Legacy macro has a parameter that isn't a valid starlark `attr` type. + +Pull as much logic as possible into a nested symbolic macro, but keep the +top level macro a legacy macro. + +* Legacy macro calls a rule that creates a target that breaks the naming schema + +That's okay, just don't depend on the "offending" target. The naming check will +be quietly ignored. + diff --git a/external/extension.mdx b/external/extension.mdx new file mode 100644 index 0000000..ababc7f --- /dev/null +++ b/external/extension.mdx @@ -0,0 +1,311 @@ +--- +title: 'Module extensions' +--- + + + +Module extensions allow users to extend the module system by reading input data +from modules across the dependency graph, performing necessary logic to resolve +dependencies, and finally creating repos by calling [repo +rules](/external/repo). These extensions have capabilities similar to repo +rules, which enables them to perform file I/O, send network requests, and so on. +Among other things, they allow Bazel to interact with other package management +systems while also respecting the dependency graph built out of Bazel modules. + +You can define module extensions in `.bzl` files, just like repo rules. They're +not invoked directly; rather, each module specifies pieces of data called *tags* +for extensions to read. Bazel runs module resolution before evaluating any +extensions. The extension reads all the tags belonging to it across the entire +dependency graph. + +## Extension usage + +Extensions are hosted in Bazel modules themselves. To use an extension in a +module, first add a `bazel_dep` on the module hosting the extension, and then +call the [`use_extension`](/rules/lib/globals/module#use_extension) built-in function +to bring it into scope. Consider the following example — a snippet from a +`MODULE.bazel` file to use the "maven" extension defined in the +[`rules_jvm_external`](https://github.com/bazelbuild/rules_jvm_external) +module: + +```python +bazel_dep(name = "rules_jvm_external", version = "4.5") +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +``` + +This binds the return value of `use_extension` to a variable, which allows the +user to use dot-syntax to specify tags for the extension. The tags must follow +the schema defined by the corresponding *tag classes* specified in the +[extension definition](#extension_definition). For an example specifying some +`maven.install` and `maven.artifact` tags: + +```python +maven.install(artifacts = ["org.junit:junit:4.13.2"]) +maven.artifact(group = "com.google.guava", + artifact = "guava", + version = "27.0-jre", + exclusions = ["com.google.j2objc:j2objc-annotations"]) +``` + +Use the [`use_repo`](/rules/lib/globals/module#use_repo) directive to bring repos +generated by the extension into the scope of the current module. + +```python +use_repo(maven, "maven") +``` + +Repos generated by an extension are part of its API. In this example, the +"maven" module extension promises to generate a repo called `maven`. With the +declaration above, the extension properly resolves labels such as +`@maven//:org_junit_junit` to point to the repo generated by the "maven" +extension. + +Note: Module extensions are evaluated lazily. This means that an extension will +typically not be evaluated unless some module brings one of its repositories +into scope using `use_repo` and that repository is referenced in a build. While +testing a module extension, `bazel mod deps` can be useful as it +unconditionally evaluates all module extensions. + +## Extension definition + +You can define module extensions similarly to [repo rules](/external/repo), +using the [`module_extension`](/rules/lib/globals/bzl#module_extension) +function. However, while repo rules have a number of attributes, module +extensions have [`tag_class`es](/rules/lib/globals/bzl#tag_class), each of which +has a number of attributes. The tag classes define schemas for tags used by this +extension. For example, the "maven" extension above might be defined like this: + +```python +# @rules_jvm_external//:extensions.bzl + +_install = tag_class(attrs = {"artifacts": attr.string_list(), ...}) +_artifact = tag_class(attrs = {"group": attr.string(), "artifact": attr.string(), ...}) +maven = module_extension( + implementation = _maven_impl, + tag_classes = {"install": _install, "artifact": _artifact}, +) +``` + +These declarations show that `maven.install` and `maven.artifact` tags can be +specified using the specified attribute schema. + +The implementation function of module extensions are similar to those of repo +rules, except that they get a [`module_ctx`](/rules/lib/builtins/module_ctx) object, +which grants access to all modules using the extension and all pertinent tags. +The implementation function then calls repo rules to generate repos. + +```python +# @rules_jvm_external//:extensions.bzl + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") # a repo rule +def _maven_impl(ctx): + # This is a fake implementation for demonstration purposes only + + # collect artifacts from across the dependency graph + artifacts = [] + for mod in ctx.modules: + for install in mod.tags.install: + artifacts += install.artifacts + artifacts += [_to_artifact(artifact) for artifact in mod.tags.artifact] + + # call out to the coursier CLI tool to resolve dependencies + output = ctx.execute(["coursier", "resolve", artifacts]) + repo_attrs = _process_coursier_output(output) + + # call repo rules to generate repos + for attrs in repo_attrs: + http_file(**attrs) + _generate_hub_repo(name = "maven", repo_attrs) +``` + +### Extension identity + +Module extensions are identified by the name and the `.bzl` file that appears +in the call to `use_extension`. In the following example, the extension `maven` +is identified by the `.bzl` file `@rules_jvm_external//:extension.bzl` and the +name `maven`: + +```python +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +``` + +Re-exporting an extension from a different `.bzl` file gives it a new identity +and if both versions of the extension are used in the transitive module graph, +then they will be evaluated separately and will only see the tags associated +with that particular identity. + +As an extension author you should make sure that users will only use your +module extension from one single `.bzl` file. + +## Repository names and visibility + +Repos generated by extensions have canonical names in the form of `{{ "" +}}module_repo_canonical_name+extension_name{{ +"" }}+repo_name`. Note that the canonical name +format is not an API you should depend on — it's subject to change at any time. + +This naming policy means that each extension has its own "repo namespace"; two +distinct extensions can each define a repo with the same name without risking +any clashes. It also means that `repository_ctx.name` reports the canonical name +of the repo, which is *not* the same as the name specified in the repo rule +call. + +Taking repos generated by module extensions into consideration, there are +several repo visibility rules: + +* A Bazel module repo can see all repos introduced in its `MODULE.bazel` file + via [`bazel_dep`](/rules/lib/globals/module#bazel_dep) and + [`use_repo`](/rules/lib/globals/module#use_repo). +* A repo generated by a module extension can see all repos visible to the + module that hosts the extension, *plus* all other repos generated by the + same module extension (using the names specified in the repo rule calls as + their apparent names). + * This might result in a conflict. If the module repo can see a repo with + the apparent name `foo`, and the extension generates a repo with the + specified name `foo`, then for all repos generated by that extension + `foo` refers to the former. +* Similarly, in a module extension's implementation function, repos created + by the extension can refer to each other by their apparent names in + attributes, regardless of the order in which they are created. + * In case of a conflict with a repository visible to the module, labels + passed to repository rule attributes can be wrapped in a call to + [`Label`](/rules/lib/toplevel/attr#label) to ensure that they refer to + the repo visible to the module instead of the extension-generated repo + of the same name. + +### Overriding and injecting module extension repos + +The root module can use +[`override_repo`](/rules/lib/globals/module#override_repo) and +[`inject_repo`](/rules/lib/globals/module#inject_repo) to override or inject +module extension repos. + +#### Example: Replacing `rules_java`'s `java_tools` with a vendored copy + +```python +# MODULE.bazel +local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository") +local_repository( + name = "my_java_tools", + path = "vendor/java_tools", +) + +bazel_dep(name = "rules_java", version = "7.11.1") +java_toolchains = use_extension("@rules_java//java:extension.bzl", "toolchains") + +override_repo(java_toolchains, remote_java_tools = "my_java_tools") +``` + +#### Example: Patch a Go dependency to depend on `@zlib` instead of the system zlib + +```python +# MODULE.bazel +bazel_dep(name = "gazelle", version = "0.38.0") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3") + +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") +go_deps.module_override( + patches = [ + "//patches:my_module_zlib.patch", + ], + path = "example.com/my_module", +) +use_repo(go_deps, ...) + +inject_repo(go_deps, "zlib") +``` + +```diff +# patches/my_module_zlib.patch +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -1,6 +1,6 @@ + go_binary( + name = "my_module", + importpath = "example.com/my_module", + srcs = ["my_module.go"], +- copts = ["-lz"], ++ cdeps = ["@zlib"], + ) +``` + +## Best practices + +This section describes best practices when writing extensions so they are +straightforward to use, maintainable, and adapt well to changes over time. + +### Put each extension in a separate file + +When extensions are in a different files, it allows one extension to load +repositories generated by another extension. Even if you don't use this +functionality, it's best to put them in separate files in case you need it +later. This is because the extension's identify is based on its file, so moving +the extension into another file later changes your public API and is a backwards +incompatible change for your users. + +### Specify reproducibility and use facts + +If your extension always defines the same repositories given the same inputs +(extension tags, files it reads, etc.) and in particular doesn't rely on +any [downloads](/rules/lib/builtins/module_ctx#download) that aren't guarded by +a checksum, consider returning +[`extension_metadata`](/rules/lib/builtins/module_ctx#extension_metadata) with +`reproducible = True`. This allows Bazel to skip this extension when writing to +the `MODULE.bazel` lockfile, which helps keep the lockfile small and reduces +the chance of merge conflicts. Note that Bazel still caches the results of +reproducible extensions in a way that persists across server restarts, so even +a long-running extension can be marked as reproducible without a performance +penalty. + +If your extension relies on effectively immutable data obtained from outside +the build, most commonly from the network, but you don't have a checksum +available to guard the download, consider using the `facts` parameter of +[`extension_metadata`](/rules/lib/builtins/module_ctx#extension_metadata) to +persistently record such data and thus allow your extension to become +reproducible. `facts` is expected to be a dictionary with string keys and +arbitrary JSON-like Starlark values that is always persisted in the lockfile and +available to future evaluations of the extension via the +[`facts`](/rules/lib/builtins/module_ctx#facts) field of `module_ctx`. + +`facts` are not invalidated even when the code of your module extension changes, +so be prepared to handle the case where the structure of `facts` changes. +Bazel also assumes that two different `facts` dicts produced by two different +evaluations of the same extension can be shallowly merged (i.e., as if by using +the `|` operator on two dicts). This is partially enforced by `module_ctx.facts` +not supporting enumeration of its entries, just lookups by key. + +An example of using `facts` would be to record a mapping from version numbers of +some SDK to the an object containing the download URL and checksum of that +version. The first time the extension is evaluated, it can fetch this mapping +from the network, but on later evaluations it can use the mapping from `facts` +to avoid the network requests. + +### Specify dependence on operating system and architecture + +If your extension relies on the operating system or its architecture type, +ensure to indicate this in the extension definition using the `os_dependent` +and `arch_dependent` boolean attributes. This ensures that Bazel recognizes the +need for re-evaluation if there are changes to either of them. + +Since this kind of dependence on the host makes it more difficult to maintain +the lockfile entry for this extension, consider +[marking the extension reproducible](#specify_reproducibility) if possible. + +### Only the root module should directly affect repository names + +Remember that when an extension creates repositories, they are created within +the namespace of the extension. This means collisions can occur if different +modules use the same extension and end up creating a repository with the same +name. This often manifests as a module extension's `tag_class` having a `name` +argument that is passed as a repository rule's `name` value. + +For example, say the root module, `A`, depends on module `B`. Both modules +depend on module `mylang`. If both `A` and `B` call +`mylang.toolchain(name="foo")`, they will both try to create a repository named +`foo` within the `mylang` module and an error will occur. + +To avoid this, either remove the ability to set the repository name directly, +or only allow the root module to do so. It's OK to allow the root module this +ability because nothing will depend on it, so it doesn't have to worry about +another module creating a conflicting name. diff --git a/external/faq.mdx b/external/faq.mdx new file mode 100644 index 0000000..a0239cb --- /dev/null +++ b/external/faq.mdx @@ -0,0 +1,329 @@ +--- +title: 'Frequently asked questions' +--- + + + + + +This page answers some frequently asked questions about external dependencies in +Bazel. + +## MODULE.bazel + +### How should I version a Bazel module? + +Setting `version` with the [`module`] directive in the source archive +`MODULE.bazel` can have several downsides and unintended side effects if not +managed carefully: + +* Duplication: releasing a new version of a module typically involves both + incrementing the version in `MODULE.bazel` and tagging the release, two + separate steps that can fall out of sync. While automation can + reduce this risk, it's simpler and safer to avoid it altogether. + +* Inconsistency: users overriding a module with a specific commit using a + [non-registry override] will see an incorrect version. for example, if the + `MODULE.bazel` in the source archive sets `version = "0.3.0"` but + additional commits have been made since that release, a user overriding + with one of those commits would still see `0.3.0`. In reality, the version + should reflect that it's ahead of the release, for example `0.3.1-rc1`. + +* Non-registry override issues: using placeholder values can cause issues + when users override a module with a non-registry override. For example, + `0.0.0` doesn't sort as the highest version, which is usually the expected + behavior users want when doing a non-registry override. + +Thus, it's best to avoid setting the version in the source archive +`MODULE.bazel`. Instead, set it in the `MODULE.bazel` stored in the registry +(e.g., the [Bazel Central Registry]), which is the actual source of truth for +the module version during Bazel's external dependency resolution (see [Bazel +registries]). + +This is usually automated, for example the [`rules-template`] example rule +repository uses a [bazel-contrib/publish-to-bcr publish.yaml GitHub Action] to +publish the release to the BCR. The action [generates a patch for the source +archive `MODULE.bazel`] with the release version. This patch is stored in the +registry and is applied when the module is fetched during Bazel's external +dependency resolution. + +This way, the version in the releases in the registry will be correctly set to +the released version and thus, `bazel_dep`, `single_version_override` and +`multiple_version_override` will work as expected, while avoiding potential +issues when doing a non-registry override because the version in the source +archive will be the default value (`''`), which will always be handled +correctly (it's the default version value after all) and will behave as +expected when sorting (the empty string is treated as the highest version). + +[Bazel Central Registry]: https://registry.bazel.build/ +[Bazel registries]: https://bazel.build/external/registry +[bazel-contrib/publish-to-bcr publish.yaml GitHub Action]: https://github.com/bazel-contrib/publish-to-bcr/blob/v0.2.2/.github/workflows/publish.yaml +[generates a patch for the source archive `MODULE.bazel`]: https://github.com/bazel-contrib/publish-to-bcr/blob/v0.2.2/src/domain/create-entry.ts#L176-L216 +[`module`]: /rules/lib/globals/module#module +[non-registry override]: module.md#non-registry_overrides +[`rules-template`]: https://github.com/bazel-contrib/rules-template + +### When should I increment the compatibility level? + +The [`compatibility_level`](module.md#compatibility_level) of a Bazel module +should be incremented _in the same commit_ that introduces a backwards +incompatible ("breaking") change. + +However, Bazel can throw an error if it detects that versions of the _same +module_ with _different compatibility levels_ exist in the resolved dependency +graph. This can happen when for example' two modules depend on versions of a +third module with different compatibility levels. + +Thus, incrementing `compatibility_level` too frequently can be very disruptive +and is discouraged. To avoid this situation, the `compatibility_level` should be +incremented _only_ when the breaking change affects most use cases and isn't +easy to migrate and/or work-around. + +### Why does MODULE.bazel not support `load`s? + +During dependency resolution, the MODULE.bazel file of all referenced external +dependencies are fetched from registries. At this stage, the source archives of +the dependencies are not fetched yet; so if the MODULE.bazel file `load`s +another file, there is no way for Bazel to actually fetch that file without +fetching the entire source archive. Note the MODULE.bazel file itself is +special, as it's directly hosted on the registry. + +There are a few use cases that people asking for `load`s in MODULE.bazel are +generally interested in, and they can be solved without `load`s: + +* Ensuring that the version listed in MODULE.bazel is consistent with build + metadata stored elsewhere, for example in a .bzl file: This can be achieved + by using the + [`native.module_version`](/rules/lib/toplevel/native#module_version) method + in a .bzl file loaded from a BUILD file. +* Splitting up a very large MODULE.bazel file into manageable sections, + particularly for monorepos: The root module can use the + [`include`](/rules/lib/globals/module#include) directive to split its + MODULE.bazel file into multiple segments. For the same reason we don't allow + `load`s in MODULE.bazel files, `include` cannot be used in non-root modules. +* Users of the old WORKSPACE system might remember declaring a repo, and then + immediately `load`ing from that repo to perform complex logic. This + capability has been replaced by [module extensions](extension). + +### Can I specify a SemVer range for a `bazel_dep`? + +No. Some other package managers like [npm][npm-semver] and [Cargo][cargo-semver] +support version ranges (implicitly or explicitly), and this often requires a +constraint solver (making the output harder to predict for users) and makes +version resolution nonreproducible without a lockfile. + +Bazel instead uses [Minimal Version Selection](module#version-selection) like +Go, which in contrast makes the output easy to predict and guarantees +reproducibility. This is a tradeoff that matches Bazel's design goals. + +Furthermore, Bazel module versions are [a superset of +SemVer](module#version-format), so what makes sense in a strict SemVer +environment doesn't always carry over to Bazel module versions. + +### Can I automatically get the latest version for a `bazel_dep`? + +Some users occasionally ask for the ability to specify `bazel_dep(name = "foo", +version = "latest")` to automatically get the latest version of a dep. This is +similar to [the question about SemVer +ranges](#can-i-specify-a-semver-range-for-a-bazel-dep), and the answer is also +no. + +The recommended solution here is to have automation take care of this. For +example, [Renovate](https://docs.renovatebot.com/modules/manager/) supports +Bazel modules. + +Sometimes, users asking this question are really looking for a way to quickly +iterate during local development. This can be achieved by using a +[`local_path_override`](/rules/lib/globals/module#local_path_override). + +### Why all these `use_repo`s? + +Module extension usages in MODULE.bazel files sometimes come with a big +`use_repo` directive. For example, a typical usage of the +[`go_deps` extension][go_deps] from `gazelle` might look like: + +```python +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") +use_repo( + go_deps, + "com_github_gogo_protobuf", + "com_github_golang_mock", + "com_github_golang_protobuf", + "org_golang_x_net", + ... # potentially dozens of lines... +) +``` + +The long `use_repo` directive may seem redundant, since the information is +arguably already in the referenced `go.mod` file. + +The reason Bazel needs this `use_repo` directive is that it runs module +extensions lazily. That is, a module extension is only run if its result is +observed. Since a module extension's "output" is repo definitions, this means +that we only run a module extension if a repo it defines is requested (for +instance, if the target `@org_golang_x_net//:foo` is built, in the example +above). However, we don't know which repos a module extension would define until +after we run it. This is where the `use_repo` directive comes in; the user can +tell Bazel which repos they expect the extension to generate, and Bazel would +then only run the extension when these specific repos are used. + +To help the maintain this `use_repo` directive, a module extension can return +an [`extension_metadata`](/rules/lib/builtins/module_ctx#extension_metadata) +object from its implementation function. The user can run the `bazel mod tidy` +command to update the `use_repo` directives for these module extensions. + +## Bzlmod migration + +### Which is evaluated first, MODULE.bazel or WORKSPACE? + +When both `--enable_bzlmod` and `--enable_workspace` are set, it's natural to +wonder which system is consulted first. The short answer is that MODULE.bazel +(Bzlmod) is evaluated first. + +The long answer is that "which evaluates first" is not the right question to +ask; rather, the right question to ask is: in the context of the repo with +[canonical name](overview#canonical-repo-name) `@@foo`, what does the [apparent +repo name](overview#apparent-repo-name) `@bar` resolve to? Alternatively, what +is the repo mapping of `@@base`? + +Labels with apparent repo names (a single leading `@`) can refer to different +things based on the context they're resolved from. When you see a label +`@bar//:baz` and wonder what it actually points to, you need to first find out +what the context repo is: for example, if the label is in a BUILD file located +in the repo `@@foo`, then the context repo is `@@foo`. + +Then, depending on what the context repo is, the ["repository +visibility" table](migration#repository-visibility) in the migration guide can +be used to find out which repo an apparent name actually resolves to. + +* If the context repo is the main repo (`@@`): + 1. If `bar` is an apparent repo name introduced by the root module's + MODULE.bazel file (through any of + [`bazel_dep`](/rules/lib/globals/module#bazel_dep.repo_name), + [`use_repo`](/rules/lib/globals/module#use_repo), + [`module`](/rules/lib/globals/module#module.repo_name), + [`use_repo_rule`](/rules/lib/globals/module#use_repo_rule)), then `@bar` + resolves to what that MODULE.bazel file claims. + 2. Otherwise, if `bar` is a repo defined in WORKSPACE (which means that its + canonical name is `@@bar`), then `@bar` resolves to `@@bar`. + 3. Otherwise, `@bar` resolves to something like + `@@[unknown repo 'bar' requested from @@]`, and this will ultimately + result in an error. +* If the context repo is a Bzlmod-world repo (that is, it corresponds to a + non-root Bazel module, or is generated by a module extension), then it + will only ever see other Bzlmod-world repos, and no WORKSPACE-world repos. + * Notably, this includes any repos introduced in a `non_module_deps`-like + module extension in the root module, or `use_repo_rule` instantiations + in the root module. +* If the context repo is defined in WORKSPACE: + 1. First, check if the context repo definition has the magical + `repo_mapping` attribute. If so, go through the mapping first (so for a + repo defined with `repo_mapping = {"@bar": "@baz"}`, we would be looking + at `@baz` below). + 2. If `bar` is an apparent repo name introduced by the root module's + MODULE.bazel file, then `@bar` resolves to what that MODULE.bazel file + claims. (This is the same as item 1 in the main repo case.) + 3. Otherwise, `@bar` resolves to `@@bar`. This most likely will point to a + repo `bar` defined in WORKSPACE; if such a repo is not defined, Bazel + will throw an error. + +For a more succinct version: + +* Bzlmod-world repos (excluding the main repo) will only see Bzlmod-world + repos. +* WORKSPACE-world repos (including the main repo) will first see what the root + module in the Bzlmod world defines, then fall back to seeing WORKSPACE-world + repos. + +Of note, labels in the Bazel command line (including Starlark flags, label-typed +flag values, and build/test target patterns) are treated as having the main repo +as the context repo. + +## Other + +### How do I prepare and run an offline build? + +Use the `bazel fetch` command to prefetch repos. You can use the `--repo` flag +(like `bazel fetch --repo @foo`) to fetch only the repo `@foo` (resolved in the +context of the main repo, see [question +above](#which-is-evaluated-first-module-bazel-or-workspace)), or use a target +pattern (like `bazel fetch @foo//:bar`) to fetch all transitive dependencies of +`@foo//:bar` (this is equivalent to `bazel build --nobuild @foo//:bar`). + +The make sure no fetches happen during a build, use `--nofetch`. More precisely, +this makes any attempt to run a non-local repository rule fail. + +If you want to fetch repos _and_ modify them to test locally, consider using +the [`bazel vendor`](vendor) command. + +### How do I use HTTP proxies? + +Bazel respects the `http_proxy` and `HTTPS_PROXY` environment variables commonly +accepted by other programs, such as +[curl](https://everything.curl.dev/usingcurl/proxies/env.html). + +### How do I make Bazel prefer IPv6 in dual-stack IPv4/IPv6 setups? + +On IPv6-only machines, Bazel can download dependencies with no changes. However, +on dual-stack IPv4/IPv6 machines Bazel follows the same convention as Java, +preferring IPv4 if enabled. In some situations, for example when the IPv4 +network cannot resolve/reach external addresses, this can cause `Network +unreachable` exceptions and build failures. In these cases, you can override +Bazel's behavior to prefer IPv6 by using the +[`java.net.preferIPv6Addresses=true` system +property](https://docs.oracle.com/javase/8/docs/api/java/net/doc-files/net-properties.html). +Specifically: + +* Use `--host_jvm_args=-Djava.net.preferIPv6Addresses=true` [startup + option](/docs/user-manual#startup-options), for example by adding the + following line in your [`.bazelrc` file](/run/bazelrc): + + `startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true` + +* When running Java build targets that need to connect to the internet (such + as for integration tests), use the + `--jvmopt=-Djava.net.preferIPv6Addresses=true` [tool + flag](/docs/user-manual#jvmopt). For example, include in your [`.bazelrc` + file](/run/bazelrc): + + `build --jvmopt=-Djava.net.preferIPv6Addresses` + +* If you are using + [`rules_jvm_external`](https://github.com/bazelbuild/rules_jvm_external) for + dependency version resolution, also add + `-Djava.net.preferIPv6Addresses=true` to the `COURSIER_OPTS` environment + variable to [provide JVM options for + Coursier](https://github.com/bazelbuild/rules_jvm_external#provide-jvm-options-for-coursier-with-coursier_opts). + +### Can repo rules be run remotely with remote execution? + +No; or at least, not yet. Users employing remote execution services to speed up +their builds may notice that repo rules are still run locally. For example, an +`http_archive` would be first downloaded onto the local machine (using any local +download cache if applicable), extracted, and then each source file would be +uploaded to the remote execution service as an input file. It's natural to ask +why the remote execution service doesn't just download and extract that archive, +saving a useless roundtrip. + +Part of the reason is that repo rules (and module extensions) are akin to +"scripts" that are run by Bazel itself. A remote executor doesn't necessarily +even have a Bazel installed. + +Another reason is that Bazel often needs the BUILD files in the downloaded and +extracted archives to perform loading and analysis, which _are_ performed +locally. + +There are preliminary ideas to solve this problem by re-imagining repo rules as +build rules, which would naturally allow them to be run remotely, but conversely +raise new architectural concerns (for example, the `query` commands would +potentially need to run actions, complicating their design). + +For more previous discussion on this topic, see [A way to support repositories +that need Bazel for being +fetched](https://github.com/bazelbuild/bazel/discussions/20464). + +[npm-semver]: https://docs.npmjs.com/about-semantic-versioning +[cargo-semver]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#version-requirement-syntax +[go_deps]: https://github.com/bazel-contrib/rules_go/blob/master/docs/go/core/bzlmod.md#specifying-external-dependencies + diff --git a/external/migration.mdx b/external/migration.mdx new file mode 100644 index 0000000..8e70b1f --- /dev/null +++ b/external/migration.mdx @@ -0,0 +1,862 @@ +keywords: bzlmod +--- +title: 'Bzlmod Migration Guide' +--- + + + +Due to the [shortcomings of +WORKSPACE](/external/overview#workspace-shortcomings), Bzlmod is replacing the +legacy WORKSPACE system. The WORKSPACE file is already disabled in Bazel 8 (late +2024) and will be removed in Bazel 9 (late 2025). This guide helps you migrate +your project to Bzlmod and drop WORKSPACE for managing external dependencies. + +## Why migrate to Bzlmod? + +* There are many [advantages](overview#benefits) compared to the legacy + WORKSPACE system, which helps to ensure a healthy growth of the Bazel + ecosystem. + +* If your project is a dependency of other projects, migrating to Bzlmod will + unblock their migration and make it easier for them to depend on your + project. + +* Migration to Bzlmod is a necessary step in order to use future Bazel + versions (mandatory in Bazel 9). + +## How to migrate to Bzlmod? + +Recommended migration process: + +1. Use [migration tool](/external/migration_tool) as a helper tool to + streamline the migration process as much as possible. +2. If there are errors left after using the migration tool, resolve them + manually. For understanding the main differences between concepts inside + `WORKSPACE` and `MODULE.bazel` files, check [WORKSPACE versus + Bzlmod](#workspace-vs-bzlmod) section. + +## WORKSPACE vs Bzlmod + +Bazel's WORKSPACE and Bzlmod offer similar features with different syntax. This +section explains how to migrate from specific WORKSPACE functionalities to +Bzlmod. + +### Define the root of a Bazel workspace + +The WORKSPACE file marks the source root of a Bazel project, this responsibility +is replaced by MODULE.bazel in Bazel version 6.3 and later. With Bazel versions +prior to 6.3, there should still be a `WORKSPACE` or `WORKSPACE.bazel` file at +your workspace root, maybe with comments like: + +* **WORKSPACE** + + ```python + # This file marks the root of the Bazel workspace. + # See MODULE.bazel for external dependencies setup. + ``` + +### Enable Bzlmod in your bazelrc + +`.bazelrc` lets you set flags that apply every time your run Bazel. To enable +Bzlmod, use the `--enable_bzlmod` flag, and apply it to the `common` command so +it applies to every command: + +* **.bazelrc** + + ``` + # Enable Bzlmod for every Bazel command + common --enable_bzlmod + ``` + +### Specify repository name for your workspace + +* **WORKSPACE** + + The [`workspace`](/rules/lib/globals/workspace#workspace) function is used + to specify a repository name for your workspace. This allows a target + `//foo:bar` in the workspace to be referenced as `@//foo:bar`. If not specified, the default repository name for your + workspace is `__main__`. + + ```python + ## WORKSPACE + workspace(name = "com_foo_bar") + ``` + +* **Bzlmod** + + It's recommended to reference targets in the same workspace with the + `//foo:bar` syntax without `@`. But if you do need the old syntax + , you can use the module name specified by the + [`module`](/rules/lib/globals/module#module) function as the repository + name. If the module name is different from the needed repository name, you + can use `repo_name` attribute of the + [`module`](/rules/lib/globals/module#module) function to override the + repository name. + + ```python + ## MODULE.bazel + module( + name = "bar", + repo_name = "com_foo_bar", + ) + ``` + +### Fetch external dependencies as Bazel modules + +If your dependency is a Bazel project, you should be able to depend on it as a +Bazel module when it also adopts Bzlmod. + +* **WORKSPACE** + + With WORKSPACE, it's common to use the + [`http_archive`](/rules/lib/repo/http#http_archive) or + [`git_repository`](/rules/lib/repo/git#git_repository) repository rules to + download the sources of the Bazel project. + + ```python + ## WORKSPACE + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.2/bazel-skylib-1.4.2.tar.gz"], + sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", + ) + load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + bazel_skylib_workspace() + + http_archive( + name = "rules_java", + urls = ["https://github.com/bazelbuild/rules_java/releases/download/6.1.1/rules_java-6.1.1.tar.gz"], + sha256 = "76402a50ae6859d50bd7aed8c1b8ef09dae5c1035bb3ca7d276f7f3ce659818a", + ) + load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains") + rules_java_dependencies() + rules_java_toolchains() + ``` + + As you can see, it's a common pattern that users need to load transitive + dependencies from a macro of the dependency. Assume both `bazel_skylib` and + `rules_java` depends on `platform`, the exact version of the `platform` + dependency is determined by the order of the macros. + +* **Bzlmod** + + With Bzlmod, as long as your dependency is available in [Bazel Central + Registry](https://registry.bazel.build) or your custom [Bazel + registry](/external/registry), you can simply depend on it with a + [`bazel_dep`](/rules/lib/globals/module#bazel_dep) directive. + + ```python + ## MODULE.bazel + bazel_dep(name = "bazel_skylib", version = "1.4.2") + bazel_dep(name = "rules_java", version = "6.1.1") + ``` + + Bzlmod resolves Bazel module dependencies transitively using the + [MVS](https://research.swtch.com/vgo-mvs) algorithm. Therefore, the maximal + required version of `platform` is selected automatically. + +### Override a dependency as a Bazel module + +As the root module, you can override Bazel module dependencies in different +ways. + +Please read the [overrides](/external/module#overrides) section for more +information. + +You can find some example usages in the +[examples][override-examples] +repository. + +[override-examples]: https://github.com/bazelbuild/examples/blob/main/bzlmod/02-override_bazel_module + +### Fetch external dependencies with module extensions + +If your dependency is not a Bazel project or not yet available in any Bazel +registry, you can introduce it using +[`use_repo_rule`](/external/module#use_repo_rule) or [module +extensions](/external/extension). + +* **WORKSPACE** + + Download a file using the [`http_file`](/rules/lib/repo/http#http_file) + repository rule. + + ```python + ## WORKSPACE + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + +* **Bzlmod** + + With Bzlmod, you can use the `use_repo_rule` directive in your MODULE.bazel + file to directly instantiate repos: + + ```python + ## MODULE.bazel + http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + + Under the hood, this is implemented using a module extension. If you need to + perform more complex logic than simply invoking a repo rule, you could also + implement a module extension yourself. You'll need to move the definition + into a `.bzl` file, which also lets you share the definition between + WORKSPACE and Bzlmod during the migration period. + + ```python + ## repositories.bzl + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + def my_data_dependency(): + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + + Implement a module extension to load the dependencies macro. You can define + it in the same `.bzl` file of the macro, but to keep compatibility with + older Bazel versions, it's better to define it in a separate `.bzl` file. + + ```python + ## extensions.bzl + load("//:repositories.bzl", "my_data_dependency") + def _non_module_dependencies_impl(_ctx): + my_data_dependency() + + non_module_dependencies = module_extension( + implementation = _non_module_dependencies_impl, + ) + ``` + + To make the repository visible to the root project, you should declare the + usages of the module extension and the repository in the MODULE.bazel file. + + ```python + ## MODULE.bazel + non_module_dependencies = use_extension("//:extensions.bzl", "non_module_dependencies") + use_repo(non_module_dependencies, "data_file") + ``` + +### Resolve conflict external dependencies with module extension + +A project can provide a macro that introduces external repositories based on +inputs from its callers. But what if there are multiple callers in the +dependency graph and they cause a conflict? + +Assume the project `foo` provides the following macro which takes `version` as +an argument. + +```python +## repositories.bzl in foo {:#repositories.bzl-foo} +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +def data_deps(version = "1.0"): + http_file( + name = "data_file", + url = "http://example.com/file-%s" % version, + # Omitting the "sha256" attribute for simplicity + ) +``` + +* **WORKSPACE** + + With WORKSPACE, you can load the macro from `@foo` and specify the version + of the data dependency you need. Assume you have another dependency `@bar`, + which also depends on `@foo` but requires a different version of the data + dependency. + + ```python + ## WORKSPACE + + # Introduce @foo and @bar. + ... + + load("@foo//:repositories.bzl", "data_deps") + data_deps(version = "2.0") + + load("@bar//:repositories.bzl", "bar_deps") + bar_deps() # -> which calls data_deps(version = "3.0") + ``` + + In this case, the end user must carefully adjust the order of macros in the + WORKSPACE to get the version they need. This is one of the biggest pain + points with WORKSPACE since it doesn't really provide a sensible way to + resolve dependencies. + +* **Bzlmod** + + With Bzlmod, the author of project `foo` can use module extension to resolve + conflicts. For example, let's assume it makes sense to always select the + maximal required version of the data dependency among all Bazel modules. + + ```python + ## extensions.bzl in foo + load("//:repositories.bzl", "data_deps") + + data = tag_class(attrs={"version": attr.string()}) + + def _data_deps_extension_impl(module_ctx): + # Select the maximal required version in the dependency graph. + version = "1.0" + for mod in module_ctx.modules: + for data in mod.tags.data: + version = max(version, data.version) + data_deps(version) + + data_deps_extension = module_extension( + implementation = _data_deps_extension_impl, + tag_classes = {"data": data}, + ) + ``` + + ```python + ## MODULE.bazel in bar + bazel_dep(name = "foo", version = "1.0") + + foo_data_deps = use_extension("@foo//:extensions.bzl", "data_deps_extension") + foo_data_deps.data(version = "3.0") + use_repo(foo_data_deps, "data_file") + ``` + + ```python + ## MODULE.bazel in root module + bazel_dep(name = "foo", version = "1.0") + bazel_dep(name = "bar", version = "1.0") + + foo_data_deps = use_extension("@foo//:extensions.bzl", "data_deps_extension") + foo_data_deps.data(version = "2.0") + use_repo(foo_data_deps, "data_file") + ``` + + In this case, the root module requires data version `2.0`, while its + dependency `bar` requires `3.0`. The module extension in `foo` can correctly + resolve this conflict and automatically select version `3.0` for the data + dependency. + +### Integrate third party package manager + +Following the last section, since module extension provides a way to collect +information from the dependency graph, perform custom logic to resolve +dependencies and call repository rules to introduce external repositories, this +provides a great way for rules authors to enhance the rulesets that integrate +package managers for specific languages. + +Please read the [module extensions](/external/extension) page to learn more +about how to use module extensions. + +Here is a list of the rulesets that already adopted Bzlmod to fetch dependencies +from different package managers: + +- [rules_jvm_external](https://github.com/bazelbuild/rules_jvm_external/blob/master/docs/bzlmod.md) +- [rules_go](https://github.com/bazelbuild/rules_go/blob/master/docs/go/core/bzlmod.md) +- [rules_python](https://github.com/bazelbuild/rules_python/blob/main/BZLMOD_SUPPORT.md) + +A minimal example that integrates a pseudo package manager is available at the +[examples][pkg-mgr-example] +repository. + +[pkg-mgr-example]: https://github.com/bazelbuild/examples/tree/main/bzlmod/05-integrate_third_party_package_manager + +### Detect toolchains on the host machine + +When Bazel build rules need to detect what toolchains are available on your host +machine, they use repository rules to inspect the host machine and generate +toolchain info as external repositories. + +* **WORKSPACE** + + Given the following repository rule to detect a shell toolchain. + + ```python + ## local_config_sh.bzl + def _sh_config_rule_impl(repository_ctx): + sh_path = get_sh_path_from_env("SH_BIN_PATH") + + if not sh_path: + sh_path = detect_sh_from_path() + + if not sh_path: + sh_path = "/shell/binary/not/found" + + repository_ctx.file("BUILD", """ + load("@bazel_tools//tools/sh:sh_toolchain.bzl", "sh_toolchain") + sh_toolchain( + name = "local_sh", + path = "{sh_path}", + visibility = ["//visibility:public"], + ) + toolchain( + name = "local_sh_toolchain", + toolchain = ":local_sh", + toolchain_type = "@bazel_tools//tools/sh:toolchain_type", + ) + """.format(sh_path = sh_path)) + + sh_config_rule = repository_rule( + environ = ["SH_BIN_PATH"], + local = True, + implementation = _sh_config_rule_impl, + ) + ``` + + You can load the repository rule in WORKSPACE. + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_config_rule") + sh_config_rule(name = "local_config_sh") + ``` + +* **Bzlmod** + + With Bzlmod, you can introduce the same repository using a module extension, + which is similar to introducing the `@data_file` repository in the last + section. + + ``` + ## local_config_sh_extension.bzl + load("//:local_config_sh.bzl", "sh_config_rule") + + sh_config_extension = module_extension( + implementation = lambda ctx: sh_config_rule(name = "local_config_sh"), + ) + ``` + + Then use the extension in the MODULE.bazel file. + + ```python + ## MODULE.bazel + sh_config_ext = use_extension("//:local_config_sh_extension.bzl", "sh_config_extension") + use_repo(sh_config_ext, "local_config_sh") + ``` + +### Register toolchains & execution platforms + +Following the last section, after introducing a repository hosting toolchain +information (e.g. `local_config_sh`), you probably want to register the +toolchain. + +* **WORKSPACE** + + With WORKSPACE, you can register the toolchain in the following ways. + + 1. You can register the toolchain the `.bzl` file and load the macro in the + WORKSPACE file. + + ```python + ## local_config_sh.bzl + def sh_configure(): + sh_config_rule(name = "local_config_sh") + native.register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_configure") + sh_configure() + ``` + + 2. Or register the toolchain in the WORKSPACE file directly. + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_config_rule") + sh_config_rule(name = "local_config_sh") + register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + +* **Bzlmod** + + With Bzlmod, the + [`register_toolchains`](/rules/lib/globals/module#register_toolchains) and + [`register_execution_platforms`][register_execution_platforms] + APIs are only available in the MODULE.bazel file. You cannot call + `native.register_toolchains` in a module extension. + + ```python + ## MODULE.bazel + sh_config_ext = use_extension("//:local_config_sh_extension.bzl", "sh_config_extension") + use_repo(sh_config_ext, "local_config_sh") + register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + +The toolchains and execution platforms registered in `WORKSPACE`, +`WORKSPACE.bzlmod` and each Bazel module's `MODULE.bazel` file follow this +order of precedence during toolchain selection (from highest to lowest): + +1. toolchains and execution platforms registered in the root module's + `MODULE.bazel` file. +2. toolchains and execution platforms registered in the `WORKSPACE` or + `WORKSPACE.bzlmod` file. +3. toolchains and execution platforms registered by modules that are + (transitive) dependencies of the root module. +4. when not using `WORKSPACE.bzlmod`: toolchains registered in the `WORKSPACE` + [suffix](/external/migration#builtin-default-deps). + +[register_execution_platforms]: /rules/lib/globals/module#register_execution_platforms + +### Introduce local repositories + +You may need to introduce a dependency as a local repository when you need a +local version of the dependency for debugging or you want to incorporate a +directory in your workspace as external repository. + +* **WORKSPACE** + + With WORKSPACE, this is achieved by two native repository rules, + [`local_repository`](/reference/be/workspace#local_repository) and + [`new_local_repository`](/reference/be/workspace#new_local_repository). + + ```python + ## WORKSPACE + local_repository( + name = "rules_java", + path = "/Users/bazel_user/workspace/rules_java", + ) + ``` + +* **Bzlmod** + + With Bzlmod, you can use + [`local_path_override`](/rules/lib/globals/module#local_path_override) to + override a module with a local path. + + ```python + ## MODULE.bazel + bazel_dep(name = "rules_java") + local_path_override( + module_name = "rules_java", + path = "/Users/bazel_user/workspace/rules_java", + ) + ``` + + Note: With `local_path_override`, you can only introduce a local directory + as a Bazel module, which means it should have a MODULE.bazel file and its + transitive dependencies are taken into consideration during dependency + resolution. In addition, all module override directives can only be used by + the root module. + + It is also possible to introduce a local repository with module extension. + However, you cannot call `native.local_repository` in module extension, + there is ongoing effort on starlarkifying all native repository rules (check + [#18285](https://github.com/bazelbuild/bazel/issues/18285) for progress). + Then you can call the corresponding starlark `local_repository` in a module + extension. It's also trivial to implement a custom version of + `local_repository` repository rule if this is a blocking issue for you. + +### Bind targets + +The [`bind`](/reference/be/workspace#bind) rule in WORKSPACE is deprecated and +not supported in Bzlmod. It was introduced to give a target an alias in the +special `//external` package. All users depending on this should migrate away. + +For example, if you have + +```python +## WORKSPACE +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +``` + +This allows other targets to depend on `//external:openssl`. You can migrate +away from this by: + +* Replace all usages of `//external:openssl` with `@my-ssl//src:openssl-lib`. + * Tip: Use `bazel query --output=build --noenable_bzlmod + --enable_workspace [target]` command to find relevant info + about the target. + +* Or use the [`alias`](/reference/be/general#alias) build rule + * Define the following target in a package (e.g. `//third_party`) + + ```python + ## third_party/BUILD + alias( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", + ) + ``` + + * Replace all usages of `//external:openssl` with `//third_party:openssl`. + +### Fetch versus Sync + +Fetch and sync commands are used to download external repos locally and keep +them updated. Sometimes also to allow building offline using the `--nofetch` +flag after fetching all repos needed for a build. + +* **WORKSPACE** + + Sync performs a force fetch for all repositories, or for a specific + configured set of repos, while fetch is _only_ used to fetch for a specific + target. + +* **Bzlmod** + + The sync command is no longer applicable, but fetch offers + [various options](/reference/command-line-reference#fetch-options). + You can fetch a target, a repository, a set of configured repos or all + repositories involved in your dependency resolution and module extensions. + The fetch result is cached and to force a fetch you must include the + `--force` option during the fetch process. + +## Manual migration + +This section provides useful information and guidance for your **manual** Bzlmod +migration process. For more automatized migration process, check [recommended +migration process](#how-migrate-to-bzlmod) section. + +### Know your dependencies in WORKSPACE + +The first step of migration is to understand what dependencies you have. It +could be hard to figure out what exact dependencies are introduced in the +WORKSPACE file because transitive dependencies are often loaded with `*_deps` +macros. + +#### Inspect external dependency with workspace resolved file + +Fortunately, the flag +[`--experimental_repository_resolved_file`][resolved_file_flag] +can help. This flag essentially generates a "lock file" of all fetched external +dependencies in your last Bazel command. You can find more details in this [blog +post](https://blog.bazel.build/2018/07/09/bazel-sync-and-resolved-file.html). + +[resolved_file_flag]: /reference/command-line-reference#flag--experimental_repository_resolved_file + +It can be used in two ways: + +1. To fetch info of external dependencies needed for building certain targets. + + ```shell + bazel clean --expunge + bazel build --nobuild --experimental_repository_resolved_file=resolved.bzl //foo:bar + ``` + +2. To fetch info of all external dependencies defined in the WORKSPACE file. + + ```shell + bazel clean --expunge + bazel sync --experimental_repository_resolved_file=resolved.bzl + ``` + + With the `bazel sync` command, you can fetch all dependencies defined in the + WORKSPACE file, which include: + + * `bind` usages + * `register_toolchains` & `register_execution_platforms` usages + + However, if your project is cross platforms, bazel sync may break on certain + platforms because some repository rules may only run correctly on supported + platforms. + +After running the command, you should have information of your external +dependencies in the `resolved.bzl` file. + +#### Inspect external dependency with `bazel query` + +You may also know `bazel query` can be used for inspecting repository rules with + +```shell +bazel query --output=build //external: +``` + +While it is more convenient and much faster, [bazel query can lie about +external dependency version](https://github.com/bazelbuild/bazel/issues/12947), +so be careful using it! Querying and inspecting external +dependencies with Bzlmod is going to achieved by a [new +subcommand](https://github.com/bazelbuild/bazel/issues/15365). + +#### Built-in default dependencies + +If you check the file generated by `--experimental_repository_resolved_file`, +you are going to find many dependencies that are not defined in your WORKSPACE. +This is because Bazel in fact adds prefixes and suffixes to the user's WORKSPACE +file content to inject some default dependencies, which are usually required by +native rules (e.g. `@bazel_tools`, `@platforms` and `@remote_java_tools`). With +Bzlmod, those dependencies are introduced with a built-in module +[`bazel_tools`][bazel_tools] , which is a default dependency for every other +Bazel module. + +[bazel_tools]: https://github.com/bazelbuild/bazel/blob/master/src/MODULE.tools + +### Hybrid mode for gradual migration + +Bzlmod and WORKSPACE can work side by side, which allows migrating dependencies +from the WORKSPACE file to Bzlmod to be a gradual process. + +Note: In practice, loading "*_deps" macros in WORKSPACE often causes confusions +with Bzlmod dependencies, therefore we recommend starting with a +WORKSPACE.bzlmod file and avoid loading transitive dependencies with macros. + +#### WORKSPACE.bzlmod + +During the migration, Bazel users may need to switch between builds with and +without Bzlmod enabled. WORKSPACE.bzlmod support is implemented to make the +process smoother. + +WORKSPACE.bzlmod has the exact same syntax as WORKSPACE. When Bzlmod is enabled, +if a WORKSPACE.bzlmod file also exists at the workspace root: + +* `WORKSPACE.bzlmod` takes effect and the content of `WORKSPACE` is ignored. +* No [prefixes or suffixes](/external/migration#builtin-default-deps) are + added to the WORKSPACE.bzlmod file. + +Using the WORKSPACE.bzlmod file can make the migration easier because: + +* When Bzlmod is disabled, you fall back to fetching dependencies from the + original WORKSPACE file. +* When Bzlmod is enabled, you can better track what dependencies are left to + migrate with WORKSPACE.bzlmod. + +#### Repository visibility + +Bzlmod is able to control which other repositories are visible from a given +repository, check [repository names and strict +deps](/external/module#repository_names_and_strict_deps) for more details. + +Here is a summary of repository visibilities from different types of +repositories when also taking WORKSPACE into consideration. + +| | From the main repo | From Bazel module repos | From module extension repos | From WORKSPACE repos | +|----------------|--------------------|-------------------------|---------------------------------------------------------------------------------------------------------------------|----------------------| +| The main repo | Visible | If the root module is a direct dependency | If the root module is a direct dependency of the module hosting the module extension | Visible | +| Bazel module repos | Direct deps | Direct deps | Direct deps of the module hosting the module extension | Direct deps of the root module | +| Module extension repos | Direct deps | Direct deps | Direct deps of the module hosting the module extension + all repos generated by the same module extension | Direct deps of the root module | +| WORKSPACE Repos | All visible | Not visible | Not visible | All visible | + +Note: For the root module, if a repository `@foo` is defined in WORKSPACE and +`@foo` is also used as an [apparent repository +name](/external/overview#apparent-repo-name) in MODULE.bazel, then `@foo` +refers to the one introduced in MODULE.bazel. + +Note: For a module extension generated repository `@bar`, if `@foo` is used as +an [apparent repository name](/external/overview#apparent-repo-name) of +another repository generated by the same module extension and direct +dependencies of the module hosting the module extension, then for repository +`@bar`, `@foo` refers to the latter. + +### Manual migration process + +A typical Bzlmod migration process can look like this: + +1. Understand what dependencies you have in WORKSPACE. +1. Add an empty MODULE.bazel file at your project root. +1. Add an empty WORKSPACE.bzlmod file to override the WORKSPACE file content. +1. Build your targets with Bzlmod enabled and check which repository is + missing. +1. Check the definition of the missing repository in the resolved dependency + file. +1. Introduce the missing dependency as a Bazel module, through a module + extension, or leave it in the WORKSPACE.bzlmod for later migration. +1. Go back to 4 and repeat until all dependencies are available. + +## Publish Bazel modules + +If your Bazel project is a dependency for other projects, you can publish your +project in the [Bazel Central Registry](https://registry.bazel.build/). + +To be able to check in your project in the BCR, you need a source archive URL of +the project. Take note of a few things when creating the source archive: + +* **Make sure the archive is pointing to a specific version.** + + The BCR can only accept versioned source archives because Bzlmod needs to + conduct version comparison during dependency resolution. + +* **Make sure the archive URL is stable.** + + Bazel verifies the content of the archive by a hash value, so you should + make sure the checksum of the downloaded file never changes. If the URL is + from GitHub, please create and upload a release archive in the release page. + GitHub isn't going to guarantee the checksum of source archives generated on + demand. In short, URLs in the form of + `https://github.com///releases/download/...` is considered stable + while `https://github.com///archive/...` is not. Check [GitHub + Archive Checksum + Outage](https://blog.bazel.build/2023/02/15/github-archive-checksum.html) + for more context. + +* **Make sure the source tree follows the layout of the original repository.** + + In case your repository is very large and you want to create a distribution + archive with reduced size by stripping out unnecessary sources, please make + sure the stripped source tree is a subset of the original source tree. This + makes it easier for end users to override the module to a non-release + version by [`archive_override`](/rules/lib/globals/module#archive_override) + and [`git_override`](/rules/lib/globals/module#git_override). + +* **Include a test module in a subdirectory that tests your most common + APIs.** + + A test module is a Bazel project with its own WORKSPACE and MODULE.bazel + file located in a subdirectory of the source archive which depends on the + actual module to be published. It should contain examples or some + integration tests that cover your most common APIs. Check + [test module][test_module] to learn how to set it up. + +[test_module]: https://github.com/bazelbuild/bazel-central-registry/tree/main/docs#test-module + +When you have your source archive URL ready, follow the [BCR contribution +guidelines][bcr_contrib_guide] to submit your module to the BCR with a GitHub +Pull Request. + +[bcr_contrib_guide]: https://github.com/bazelbuild/bazel-central-registry/tree/main/docs#contribute-a-bazel-module + +It is **highly recommended** to set up the [Publish to +BCR](https://github.com/bazel-contrib/publish-to-bcr) GitHub App for your +repository to automate the process of submitting your module to the BCR. + +## Best practices + +This section documents a few best practices you should follow for better +managing your external dependencies. + +#### Split targets into different packages to avoid fetching unnecessary dependencies. + +Check [#12835](https://github.com/bazelbuild/bazel/issues/12835), where dev +dependencies for tests are forced to be fetched unnecessarily for building +targets that don't need them. This is actually not Bzlmod specific, but +following this practices makes it easier to specify dev dependencies correctly. + +#### Specify dev dependencies + +You can set the `dev_dependency` attribute to true for +[`bazel_dep`](/rules/lib/globals/module#bazel_dep) and +[`use_extension`](/rules/lib/globals/module#use_extension) directives so that +they don't propagate to dependent projects. As the root module, you can use the +[`--ignore_dev_dependency`][ignore_dev_dep_flag] flag to verify if your targets +still build without dev dependencies and overrides. + +[ignore_dev_dep_flag]: /reference/command-line-reference#flag--ignore_dev_dependency + + + +## Community migration progress + +You can check the [Bazel Central Registry](https://registry.bazel.build) to find +out if your dependencies are already available. Otherwise feel free to join this +[GitHub discussion](https://github.com/bazelbuild/bazel/discussions/18329) to +upvote or post the dependencies that are blocking your migration. + +## Report issues + +Please check the [Bazel GitHub issue list][bzlmod_github_issue] for known Bzlmod +issues. Feel free to file new issues or feature requests that can help unblock +your migration! + +[bzlmod_github_issue]: https://github.com/bazelbuild/bazel/issues?q=is%3Aopen+is%3Aissue+label%3Aarea-Bzlmod diff --git a/external/module.mdx b/external/module.mdx index 8094e4a..0499c4c 100644 --- a/external/module.mdx +++ b/external/module.mdx @@ -210,8 +210,8 @@ dependencies. This helps prevent accidental breakages due to changes in transitive dependencies. The [canonical name](/external/overview#canonical-repo-name) of a repo backing a -module is either `{{ "" }}module_name{{ "" }}+{{ "" }}version{{ -"" }}` (for example, `bazel_skylib+1.0.3`) or `{{ "" }}module_name{{ +module is either `module_name+version{{ +"" }}` (for example, `bazel_skylib+1.0.3`) or `module_name{{ "" }}+` (for example, `bazel_features+`), depending on whether there are multiple versions of the module in the entire dependency graph (see [`multiple_version_override`](/rules/lib/globals/module#multiple_version_override)). diff --git a/external/overview.mdx b/external/overview.mdx new file mode 100644 index 0000000..6e55b57 --- /dev/null +++ b/external/overview.mdx @@ -0,0 +1,305 @@ +--- +title: 'External dependencies overview' +--- + + + + +Bazel supports *external dependencies*, source files (both text and binary) used +in your build that are not from your workspace. For example, they could be a +ruleset hosted in a GitHub repo, a Maven artifact, or a directory on your local +machine outside your current workspace. + +This document gives an overview of the system before examining some of the +concepts in more detail. + +## Overview of the system + +Bazel's external dependency system works on the basis of [*Bazel +modules*](#module), each of which is a versioned Bazel project, and +[*repositories*](#repository) (or repos), which are directory trees containing +source files. + +Bazel starts from the root module -- that is, the project you're working on. +Like all modules, it needs to have a `MODULE.bazel` file at its directory root, +declaring its basic metadata and direct dependencies. The following is a basic +example: + +```python +module(name = "my-module", version = "1.0") + +bazel_dep(name = "rules_cc", version = "0.1.1") +bazel_dep(name = "platforms", version = "0.0.11") +``` + +From there, Bazel looks up all transitive dependency modules in a +[*Bazel registry*](registry) — by default, the [Bazel Central +Registry](https://bcr.bazel.build/). The registry provides the +dependencies' `MODULE.bazel` files, which allows Bazel to discover the entire +transitive dependency graph before performing version resolution. + +After version resolution, in which one version is selected for each module, +Bazel consults the registry again to learn how to define a repo for each module +-- that is, how the sources for each dependency module should be fetched. Most +of the time, these are just archives downloaded from the internet and extracted. + +Modules can also specify customized pieces of data called *tags*, which are +consumed by [*module extensions*](extension) after module resolution +to define additional repos. These extensions can perform actions like file I/O +and sending network requests. Among other things, they allow Bazel to interact +with other package management systems while also respecting the dependency graph +built out of Bazel modules. + +The three kinds of repos -- the main repo (which is the source tree you're +working in), the repos representing transitive dependency modules, and the repos +created by module extensions -- form the [*workspace*](#workspace) together. +External repos (non-main repos) are fetched on demand, for example when they're +referred to by labels (like `@repo//pkg:target`) in BUILD files. + +## Benefits + +Bazel's external dependency system offers a wide range of benefits. + +### Automatic Dependency Resolution + +- **Deterministic Version Resolution**: Bazel adopts the deterministic + [MVS](module#version-selection) version resolution algorithm, + minimizing conflicts and addressing diamond dependency issues. +- **Simplified Dependency Management**: `MODULE.bazel` declares only direct + dependencies, while transitive dependencies are automatically resolved, + providing a clearer overview of the project's dependencies. +- **[Strict Dependency visibility](module#repository_names_and_strict_deps)**: + Only direct dependencies are visible, ensuring correctness and + predictability. + +### Ecosystem Integration + +- **[Bazel Central Registry](https://registry.bazel.build/)**: A centralized + repository for discovering and managing common dependencies as Bazel + modules. +- **Adoption of Non-Bazel Projects**: When a non-Bazel project (usually a C++ + library) is adapted for Bazel and made available in BCR, it streamlines its + integration for the whole community and eliminates duplicated effort and + conflicts of custom BUILD files. +- **Unified Integration with Language-Specific Package Managers**: Rulesets + streamline integration with external package managers for non-Bazel + dependencies, including: + * [rules_jvm_external](https://github.com/bazel-contrib/rules_jvm_external/blob/master/docs/bzlmod.md) + for Maven, + * [rules_python](https://rules-python.readthedocs.io/en/latest/pypi-dependencies.html#using-bzlmod) + for PyPi, + * [bazel-gazelle](https://github.com/bazel-contrib/rules_go/blob/master/docs/go/core/bzlmod.md#external-dependencies) + for Go Modules, + * [rules_rust](https://bazelbuild.github.io/rules_rust/crate_universe_bzlmod.html) + for Cargo. + +### Advanced Features + +- **[Module Extensions](extension)**: The + [`use_repo_rule`](/rules/lib/globals/module#use_repo_rule) and module + extension features allow flexible use of custom repository rules and + resolution logic to introduce any non-Bazel dependencies. +- **[`bazel mod` Command](mod-command)**: The sub-command offers + powerful ways to inspect external dependencies. You know exactly how an + external dependency is defined and where it comes from. +- **[Vendor Mode](vendor)**: Pre-fetch the exact external dependencies you + need to facilitate offline builds. +- **[Lockfile](lockfile)**: The lockfile improves build reproducibility and + accelerates dependency resolution. +- **(Upcoming) [BCR Provenance + Attestations](https://github.com/bazelbuild/bazel-central-registry/discussions/2721)**: + Strengthen supply chain security by ensuring verified provenance of + dependencies. + +## Concepts + +This section gives more detail on concepts related to external dependencies. + +### Module + +A Bazel project that can have multiple versions, each of which can have +dependencies on other modules. + +In a local Bazel workspace, a module is represented by a repository. + +For more details, see [Bazel modules](module). + +### Repository + +A directory tree with a boundary marker file at its root, containing source +files that can be used in a Bazel build. Often shortened to just **repo**. + +A repo boundary marker file can be `MODULE.bazel` (signaling that this repo +represents a Bazel module), `REPO.bazel` (see [below](#repo.bazel)), or in +legacy contexts, `WORKSPACE` or `WORKSPACE.bazel`. Any repo boundary marker file +will signify the boundary of a repo; multiple such files can coexist in a +directory. + +### Main repository + +The repository in which the current Bazel command is being run. + +The root of the main repository is also known as the +**workspace root**. + +### Workspace + +The environment shared by all Bazel commands run in the same main repository. It +encompasses the main repo and the set of all defined external repos. + +Note that historically the concepts of "repository" and "workspace" have been +conflated; the term "workspace" has often been used to refer to the main +repository, and sometimes even used as a synonym of "repository". + +### Canonical repository name + +The name by which a repository is always addressable. Within the context of a +workspace, each repository has a single canonical name. A target inside a repo +whose canonical name is `canonical_name` can be addressed by the label +`@@canonical_name//package:target` (note the double `@`). + +The main repository always has the empty string as the canonical name. + +### Apparent repository name + +The name by which a repository is addressable in the context of a certain other +repo. This can be thought of as a repo's "nickname": The repo with the canonical +name `michael` might have the apparent name `mike` in the context of the repo +`alice`, but might have the apparent name `mickey` in the context of the repo +`bob`. In this case, a target inside `michael` can be addressed by the label +`@mike//package:target` in the context of `alice` (note the single `@`). + +Conversely, this can be understood as a **repository mapping**: each repo +maintains a mapping from "apparent repo name" to a "canonical repo name". + +### Repository rule + +A schema for repository definitions that tells Bazel how to materialize a +repository. For example, it could be "download a zip archive from a certain URL +and extract it", or "fetch a certain Maven artifact and make it available as a +`java_import` target", or simply "symlink a local directory". Every repo is +**defined** by calling a repo rule with an appropriate number of arguments. + +See [Repository rules](repo) for more information about how to write +your own repository rules. + +The most common repo rules by far are +[`http_archive`](/rules/lib/repo/http#http_archive), which downloads an archive +from a URL and extracts it, and +[`local_repository`](/reference/be/workspace#local_repository), which symlinks a +local directory that is already a Bazel repository. + +### Fetch a repository + +The action of making a repo available on local disk by running its associated +repo rule. The repos defined in a workspace are not available on local disk +before they are fetched. + +Normally, Bazel only fetches a repo when it needs something from the repo, +and the repo hasn't already been fetched. If the repo has already been fetched +before, Bazel only re-fetches it if its definition has changed. + +The `fetch` command can be used to initiate a pre-fetch for a repository, +target, or all necessary repositories to perform any build. This capability +enables offline builds using the `--nofetch` option. + +The `--fetch` option serves to manage network access. Its default value is true. +However, when set to false (`--nofetch`), the command will utilize any cached +version of the dependency, and if none exists, the command will result in +failure. + +See [fetch options](/reference/command-line-reference#fetch-options) for more +information about controlling fetch. + +### Directory layout + +After being fetched, the repo can be found in the subdirectory `external` in the +[output base](/remote/output-directories), under its canonical name. + +You can run the following command to see the contents of the repo with the +canonical name `canonical_name`: + +```posix-terminal +ls $(bazel info output_base)/external/{{ '' }} canonical_name {{ '' }} +``` + +### REPO.bazel file + +The [`REPO.bazel`](/rules/lib/globals/repo) file is used to mark the topmost +boundary of the directory tree that constitutes a repo. It doesn't need to +contain anything to serve as a repo boundary file; however, it can also be used +to specify some common attributes for all build targets inside the repo. + +The syntax of a `REPO.bazel` file is similar to `BUILD` files, except that no +`load` statements are supported. The `repo()` function takes the same arguments as the [`package()` +function](/reference/be/functions#package) in `BUILD` files; whereas `package()` +specifies common attributes for all build targets inside the package, `repo()` +analogously does so for all build targets inside the repo. + +For example, you can specify a common license for all targets in your repo by +having the following `REPO.bazel` file: + +```python +repo( + default_package_metadata = ["//:my_license"], +) +``` + +## The legacy WORKSPACE system + +In older Bazel versions (before 9.0), external dependencies were introduced by +defining repos in the `WORKSPACE` (or `WORKSPACE.bazel`) file. This file has a +similar syntax to `BUILD` files, employing repo rules instead of build rules. + +The following snippet is an example to use the `http_archive` repo rule in the +`WORKSPACE` file: + +```python +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "foo", + urls = ["https://example.com/foo.zip"], + sha256 = "c9526390a7cd420fdcec2988b4f3626fe9c5b51e2959f685e8f4d170d1a9bd96", +) +``` + +The snippet defines a repo whose canonical name is `foo`. In the `WORKSPACE` +system, by default, the canonical name of a repo is also its apparent name to +all other repos. + +See the [full list](/rules/lib/globals/workspace) of functions available in +`WORKSPACE` files. + +### Shortcomings of the `WORKSPACE` system + +In the years after the `WORKSPACE` system was introduced, users reported many +pain points, including: + +* Bazel does not evaluate the `WORKSPACE` files of any dependencies, so all + transitive dependencies must be defined in the `WORKSPACE` file of the main + repo, in addition to direct dependencies. +* To work around this, projects have adopted the "deps.bzl" pattern, in which + they define a macro which in turn defines multiple repos, and ask users to + call this macro in their `WORKSPACE` files. + * This has its own problems: macros cannot `load` other `.bzl` files, so + these projects have to define their transitive dependencies in this + "deps" macro, or work around this issue by having the user call multiple + layered "deps" macros. + * Bazel evaluates the `WORKSPACE` file sequentially. Additionally, + dependencies are specified using `http_archive` with URLs, without any + version information. This means that there is no reliable way to perform + version resolution in the case of diamond dependencies (`A` depends on + `B` and `C`; `B` and `C` both depend on different versions of `D`). + +Due to the shortcomings of WORKSPACE, the new module-based system (codenamed +"Bzlmod") gradually replaced the legacy WORKSPACE system between Bazel 6 and 9. +Read the [Bzlmod migration guide](migration) on how to migrate +to Bzlmod. + +### External links on Bzlmod + +* [Bzlmod usage examples in bazelbuild/examples](https://github.com/bazelbuild/examples/tree/main/bzlmod) +* [Bazel External Dependencies Overhaul](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit) + (original Bzlmod design doc) +* [BazelCon 2021 talk on Bzlmod](https://www.youtube.com/watch?v=TxOCKtU39Fs) +* [Bazel Community Day talk on Bzlmod](https://www.youtube.com/watch?v=MB6xxis9gWI) diff --git a/external/vendor.mdx b/external/vendor.mdx new file mode 100644 index 0000000..653c194 --- /dev/null +++ b/external/vendor.mdx @@ -0,0 +1,213 @@ +keywords: product:Bazel,Bzlmod,vendor +--- +title: 'Vendor Mode' +--- + + + +Vendor mode is a feature that lets you create a local copy of +external dependencies. This is useful for offline builds, or when you want to +control the source of an external dependency. + +## Enable vendor mode + +You can enable vendor mode by specifying `--vendor_dir` flag. + +For example, by adding it to your `.bazelrc` file: + +```none +# Enable vendor mode with vendor directory under /vendor_src +common --vendor_dir=vendor_src +``` + +The vendor directory can be either a relative path to your workspace root or an +absolute path. + +## Vendor a specific external repository + +You can use the `vendor` command with the `--repo` flag to specify which repo +to vendor, it accepts both [canonical repo +name](/external/overview#canonical-repo-name) and [apparent repo +name](/external/overview#apparent-repo-name). + +For example, running: + +```none +bazel vendor --vendor_dir=vendor_src --repo=@rules_cc +``` + +or + +```none +bazel vendor --vendor_dir=vendor_src --repo=@@rules_cc+ +``` + +will both get rules_cc to be vendored under +`/vendor_src/rules_cc+`. + +## Vendor external dependencies for given targets + +To vendor all external dependencies required for building given target patterns, +you can run `bazel vendor `. + +For example + +```none +bazel vendor --vendor_dir=vendor_src //src/main:hello-world //src/test/... +``` + +will vendor all repos required for building the `//src/main:hello-world` target +and all targets under `//src/test/...` with the current configuration. + +Under the hood, it's doing a `bazel build --nobuild` command to analyze the +target patterns, therefore build flags could be applied to this command and +affect the result. + +### Build the target offline + +With the external dependencies vendored, you can build the target offline by + +```none +bazel build --vendor_dir=vendor_src //src/main:hello-world //src/test/... +``` + +The build should work in a clean build environment without network access and +repository cache. + +Therefore, you should be able to check in the vendored source and build the same +targets offline on another machine. + +Note: If you make changes to the targets to build, the external dependencies, +the build configuration, or the Bazel version, you may need to re-vendor to make +sure offline build still works. + +## Vendor all external dependencies + +To vendor all repos in your transitive external dependencies graph, you can +run: + +```none +bazel vendor --vendor_dir=vendor_src +``` + +Note that vendoring all dependencies has a few **disadvantages**: + +- Fetching all repos, including those introduced transitively, can be time-consuming. +- The vendor directory can become very large. +- Some repos may fail to fetch if they are not compatible with the current platform or environment. + +Therefore, consider vendoring for specific targets first. + +## Configure vendor mode with VENDOR.bazel + +You can control how given repos are handled with the VENDOR.bazel file located +under the vendor directory. + +There are two directives available, both accepting a list of +[canonical repo names](/external/overview#canonical-repo-name) as arguments: + +- `ignore()`: to completely ignore a repository from vendor mode. +- `pin()`: to pin a repository to its current vendored source as if there is a + `--override_repository` flag for this repo. Bazel will NOT update the vendored + source for this repo while running the vendor command unless it's unpinned. + The user can modify and maintain the vendored source for this repo manually. + +For example + +```python +ignore("@@rules_cc+") +pin("@@bazel_skylib+") +``` + +With this configuration + +- Both repos will be excluded from subsequent vendor commands. +- Repo `bazel_skylib` will be overridden to the source located under the + vendor directory. +- The user can safely modify the vendored source of `bazel_skylib`. +- To re-vendor `bazel_skylib`, the user has to disable the pin statement + first. + +Note: Repository rules with +[`local`](/rules/lib/globals/bzl#repository_rule.local) or +[`configure`](/rules/lib/globals/bzl#repository_rule.configure) set to true are +always excluded from vendoring. + +## Understand how vendor mode works + +Bazel fetches external dependencies of a project under `$(bazel info +output_base)/external`. Vendoring external dependencies means moving out +relevant files and directories to the given vendor directory and use the +vendored source for later builds. + +The content being vendored includes: + +- The repo directory +- The repo marker file + +During a build, if the vendored marker file is up-to-date or the repo is +pinned in the VENDOR.bazel file, then Bazel uses the vendored source by creating +a symlink to it under `$(bazel info output_base)/external` instead of actually +running the repository rule. Otherwise, a warning is printed and Bazel will +fallback to fetching the latest version of the repo. + +Note: Bazel assumes the vendored source is not changed by users unless the repo +is pinned in the VENDOR.bazel file. If a user does change the vendored source +without pinning the repo, the changed vendored source will be used, but it will +be overwritten if its existing marker file is +outdated and the repo is vendored again. + +### Vendor registry files + +Bazel has to perform the Bazel module resolution in order to fetch external +dependencies, which may require accessing registry files through internet. To +achieve offline build, Bazel vendors all registry files fetched from +network under the `/_registries` directory. + +### Vendor symlinks + +External repositories may contain symlinks pointing to other files or +directories. To make sure symlinks work correctly, Bazel uses the following +strategy to rewrite symlinks in the vendored source: + +- Create a symlink `/bazel-external` that points to `$(bazel info + output_base)/external`. It is refreshed by every Bazel command + automatically. +- For the vendored source, rewrite all symlinks that originally point to a + path under `$(bazel info output_base)/external` to a relative path under + `/bazel-external`. + +For example, if the original symlink is + +```none +/repo_foo+/link => $(bazel info output_base)/external/repo_bar+/file +``` + +It will be rewritten to + +```none +/repo_foo+/link => ../../bazel-external/repo_bar+/file +``` + +where + +```none +/bazel-external => $(bazel info output_base)/external # This might be new if output base is changed +``` + +Since `/bazel-external` is generated by Bazel automatically, it's +recommended to add it to `.gitignore` or equivalent to avoid checking it in. + +With this strategy, symlinks in the vendored source should work correctly even +after the vendored source is moved to another location or the bazel output base +is changed. + +Note: symlinks that point to an absolute path outside of $(bazel info +output_base)/external are not rewritten. Therefore, it could still break +cross-machine compatibility. + +Note: On Windows, vendoring symlinks only works with +[`--windows_enable_symlinks`][windows_enable_symlinks] +flag enabled. + +[windows_enable_symlinks]: /reference/command-line-reference#flag--windows_enable_symlinks diff --git a/install/docker-container.mdx b/install/docker-container.mdx index 3a5d017..b1efb76 100644 --- a/install/docker-container.mdx +++ b/install/docker-container.mdx @@ -43,7 +43,7 @@ docker run \ build //absl/... ``` -Build the project with sanitizers by adding the `--config={{ "" }}asan{{ "" }}|{{ "" }}tsan{{ "" }}|{{ "" }}msan{{ "" }}` build +Build the project with sanitizers by adding the `--config=asan|tsan|msan` build flag to select AddressSanitizer (asan), ThreadSanitizer (tsan) or MemorySanitizer (msan) accordingly. @@ -86,7 +86,7 @@ Do a regular build. ubuntu@5a99103747c6:~/abseil-cpp$ bazel build //absl/... ``` -Build the project with sanitizers by adding the `--config={{ "" }}asan{{ "" }}|{{ "" }}tsan{{ "" }}|{{ "" }}msan{{ "" }}` +Build the project with sanitizers by adding the `--config=asan|tsan|msan` build flag to select AddressSanitizer (asan), ThreadSanitizer (tsan) or MemorySanitizer (msan) accordingly. diff --git a/install/os-x.mdx b/install/os-x.mdx index 9a0f3f8..b8d5bca 100644 --- a/install/os-x.mdx +++ b/install/os-x.mdx @@ -121,7 +121,7 @@ the installer from the terminal using `curl` as a workaround; see Step 2 above. ### Step 4: Set up your environment If you ran the Bazel installer with the `--user` flag as above, the Bazel -executable is installed in your `{{ '' }}HOME{{ '' }}/bin` directory. +executable is installed in your `HOME/bin` directory. It's a good idea to add this directory to your default paths, as follows: ```posix-terminal diff --git a/install/suse.mdx b/install/suse.mdx index a4d2e9e..741b108 100644 --- a/install/suse.mdx +++ b/install/suse.mdx @@ -27,7 +27,7 @@ zypper install bazel ``` You can also install a specific version of Bazel by specifying the package -version with `bazel{{ '' }}version{{ '' }}`. For example, to install +version with `bazelversion`. For example, to install Bazel 4.2, run: ```posix-terminal diff --git a/install/ubuntu.mdx b/install/ubuntu.mdx index a31bd2f..73d4275 100644 --- a/install/ubuntu.mdx +++ b/install/ubuntu.mdx @@ -125,7 +125,7 @@ sudo apt-get install default-jdk ### Step 2: Run the installer -Next, download the Bazel binary installer named `bazel-{{ '' }}version{{ '' }}-installer-linux-x86_64.sh` +Next, download the Bazel binary installer named `bazel-version-installer-linux-x86_64.sh` from the [Bazel releases page on GitHub](https://github.com/bazelbuild/bazel/releases). Run it as follows: diff --git a/install/windows.mdx b/install/windows.mdx new file mode 100644 index 0000000..b38c098 --- /dev/null +++ b/install/windows.mdx @@ -0,0 +1,238 @@ +--- +title: 'Installing Bazel on Windows' +--- + + + +This page describes the requirements and steps to install Bazel on Windows. +It also includes troubleshooting and other ways to install Bazel, such as +using Chocolatey or Scoop. + +## Installing Bazel + +This section covers the prerequisites, environment setup, and detailed +steps during installation on Windows. + +### Check your system + +Recommended: 64 bit Windows 10, version 1703 (Creators Update) or newer + +To check your Windows version: + +* Click the Start button. +* Type `winver` in the search box and press Enter. +* You should see the About Windows box with your Windows version information. + +### Install the prerequisites + +* [Microsoft Visual C++ Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) + +### Download Bazel + +*Recommended*: [Use Bazelisk](/install/bazelisk) + + +Alternatively you can: + +* [Download the Bazel binary (`bazel-version-windows-x86_64.exe`) from + GitHub](https://github.com/bazelbuild/bazel/releases). +* [Install Bazel from Chocolatey](#chocolately) +* [Install Bazel from Scoop](#scoop) +* [Build Bazel from source](/install/compile-source) + +### Set up your environment + +To make Bazel easily accessible from command prompts or PowerShell by default, you can rename the Bazel binary to `bazel.exe` and add it to your default paths. + +```posix-terminal +set PATH=%PATH%;{{ '' }}path to the Bazel binary{{ '' }} +``` + +You can also change your system `PATH` environment variable to make it permanent. Check out how to [set environment variables](/configure/windows#set-environment-variables). + +### Done + +"Success: You've installed Bazel." + +To check the installation is correct, try to run: + +```posix-terminal +bazel {{ '' }}version{{ '' }} +``` + +Next, you can check out more tips and guidance here: + +* [Installing compilers and language runtimes](#install-compilers) +* [Troubleshooting](#troubleshooting) +* [Best practices on Windows](/configure/windows#best-practices) +* [Tutorials](/start/#tutorials) + +## Installing compilers and language runtimes + +Depending on which languages you want to build, you will need: + +* [MSYS2 x86_64](https://www.msys2.org/) + + MSYS2 is a software distro and building platform for Windows. It contains Bash and common Unix + tools (like `grep`, `tar`, `git`). + + You will need MSYS2 to build, test, or run targets that depend on Bash. Typically these are + `genrule`, `sh_binary`, `sh_test`, but there may be more (such as Starlark rules). Bazel shows an + error if a build target needs Bash but Bazel could not locate it. + +* Common MSYS2 packages + + You will likely need these to build and run targets that depend on Bash. MSYS2 does not install + these tools by default, so you need to install them manually. Projects that depend on Bash tools in `PATH` need this step (for example TensorFlow). + + Open the MSYS2 terminal and run this command: + + ```posix-terminal + pacman -S zip unzip patch diffutils git + ``` + + Optional: If you want to use Bazel from CMD or Powershell and still be able + to use Bash tools, make sure to add + `MSYS2_INSTALL_PATH/usr/bin` to your + `PATH` environment variable. + +* [Build Tools for Visual Studio 2019](https://aka.ms/buildtools) + + You will need this to build C++ code on Windows. + + Also supported: + + * Visual C++ Build Tools 2017 (or newer) and Windows 10 SDK + +* [Java SE Development Kit 11 (JDK) for Windows x64](https://www.oracle.com/java/technologies/javase-jdk11-downloads.html) + + You will need this to build Java code on Windows. + + Also supported: Java 8, 9, and 10 + +* [Python 3.6 for Windows x86-64](https://www.python.org/downloads/windows/) + + You will need this to build Python code on Windows. + + Also supported: Python 2.7 or newer for Windows x86-64 + +## Troubleshooting + +### Bazel does not find Bash or bash.exe + +**Possible reasons**: + +* you installed MSYS2 not under the default install path + +* you installed MSYS2 i686 instead of MSYS2 x86\_64 + +* you installed MSYS instead of MSYS2 + +**Solution**: + +Ensure you installed MSYS2 x86\_64. + +If that doesn't help: + +1. Go to Start Menu > Settings. + +2. Find the setting "Edit environment variables for your account" + +3. Look at the list on the top ("User variables for <username>"), and click the "New..." + button below it. + +4. For "Variable name", enter `BAZEL_SH` + +5. Click "Browse File..." + +6. Navigate to the MSYS2 directory, then to `usr\bin` below it. + + For example, this might be `C:\msys64\usr\bin` on your system. + +7. Select the `bash.exe` or `bash` file and click OK + +8. The "Variable value" field now has the path to `bash.exe`. Click OK to close the window. + +9. Done. + + If you open a new cmd.exe or PowerShell terminal and run Bazel now, it will find Bash. + +### Bazel does not find Visual Studio or Visual C++ + +**Possible reasons**: + +* you installed multiple versions of Visual Studio + +* you installed and removed various versions of Visual Studio + +* you installed various versions of the Windows SDK + +* you installed Visual Studio not under the default install path + +**Solution**: + +1. Go to Start Menu > Settings. + +2. Find the setting "Edit environment variables for your account" + +3. Look at the list on the top ("User variables for <username>"), and click the "New..." + button below it. + +4. For "Variable name", enter `BAZEL_VC` + +5. Click "Browse Directory..." + +6. Navigate to the `VC` directory of Visual Studio. + + For example, this might be `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC` + on your system. + +7. Select the `VC` folder and click OK + +8. The "Variable value" field now has the path to `VC`. Click OK to close the window. + +9. Done. + + If you open a new cmd.exe or PowerShell terminal and run Bazel now, it will find Visual C++. + +## Other ways to install Bazel + +### Using Chocolatey + +1. Install the [Chocolatey](https://chocolatey.org) package manager + +2. Install the Bazel package: + + ```posix-terminal + choco install bazel + ``` + + This command will install the latest available version of Bazel and + its dependencies, such as the MSYS2 shell. This will not install Visual C++ + though. + +See [Chocolatey installation and package maintenance +guide](/contribute/windows-chocolatey-maintenance) for more +information about the Chocolatey package. + +### Using Scoop + +1. Install the [Scoop](https://scoop.sh/) package manager using the following PowerShell command: + + ```posix-terminal + iex (new-object net.webclient).downloadstring('https://get.scoop.sh') + ``` + +2. Install the Bazel package: + + ```posix-terminal + scoop install bazel + ``` + +See [Scoop installation and package maintenance +guide](/contribute/windows-scoop-maintenance) for more +information about the Scoop package. + +### Build from source + +To build Bazel from scratch instead of installing, see [Compiling from source](/install/compile-source). diff --git a/query/cquery.mdx b/query/cquery.mdx index 4090be9..d35ea82 100644 --- a/query/cquery.mdx +++ b/query/cquery.mdx @@ -30,11 +30,11 @@ sh_library(name = "white-ash") sh_library(name = "common-ash") config_setting( name = "excelsior", - values = {"define": "species=excelsior"}, + values = \{"define": "species=excelsior"\}, ) config_setting( name = "americana", - values = {"define": "species=americana"}, + values = \{"define": "species=americana"\}, ) EOF ``` @@ -494,7 +494,7 @@ Extract a value from a user defined Provider. ``` $ cat some_package/my_rule.bzl - MyRuleInfo = provider(fields={"color": "the name of a color"}) + MyRuleInfo = provider(fields=\{"color": "the name of a color"\}) def _my_rule_impl(ctx): ... diff --git a/query/guide.mdx b/query/guide.mdx new file mode 100644 index 0000000..7bba0fe --- /dev/null +++ b/query/guide.mdx @@ -0,0 +1,317 @@ +--- +title: 'Query guide' +--- + + + +This page covers how to get started using Bazel's query language to trace +dependencies in your code. + +For a language details and `--output` flag details, please see the +reference manuals, [Bazel query reference](/query/language) +and [Bazel cquery reference](/query/cquery). You can get help by +typing `bazel help query` or `bazel help cquery` on the +command line. + +To execute a query while ignoring errors such as missing targets, use the +`--keep_going` flag. + +## Finding the dependencies of a rule + +To see the dependencies of `//foo`, use the +`deps` function in bazel query: + +``` +$ bazel query "deps(//foo)" +//foo:foo +//foo:foo-dep +... +``` + +This is the set of all targets required to build `//foo`. + +## Tracing the dependency chain between two packages + +The library `//third_party/zlib:zlibonly` isn't in the BUILD file for +`//foo`, but it is an indirect dependency. How can +we trace this dependency path? There are two useful functions here: +`allpaths` and `somepath`. You may also want to exclude +tooling dependencies with `--notool_deps` if you care only about +what is included in the artifact you built, and not every possible job. + +To visualize the graph of all dependencies, pipe the bazel query output through + the `dot` command-line tool: + +``` +$ bazel query "allpaths(//foo, third_party/...)" --notool_deps --output graph | dot -Tsvg > /tmp/deps.svg +``` + +Note: `dot` supports other image formats, just replace `svg` with the +format identifier, for example, `png`. + +When a dependency graph is big and complicated, it can be helpful start with a single path: + +``` +$ bazel query "somepath(//foo:foo, third_party/zlib:zlibonly)" +//foo:foo +//translations/tools:translator +//translations/base:base +//third_party/py/MySQL:MySQL +//third_party/py/MySQL:_MySQL.so +//third_party/mysql:mysql +//third_party/zlib:zlibonly +``` + +If you do not specify `--output graph` with `allpaths`, +you will get a flattened list of the dependency graph. + +``` +$ bazel query "allpaths(//foo, third_party/...)" + ...many errors detected in BUILD files... +//foo:foo +//translations/tools:translator +//translations/tools:aggregator +//translations/base:base +//tools/pkg:pex +//tools/pkg:pex_phase_one +//tools/pkg:pex_lib +//third_party/python:python_lib +//translations/tools:messages +//third_party/py/xml:xml +//third_party/py/xml:utils/boolean.so +//third_party/py/xml:parsers/sgmlop.so +//third_party/py/xml:parsers/pyexpat.so +//third_party/py/MySQL:MySQL +//third_party/py/MySQL:_MySQL.so +//third_party/mysql:mysql +//third_party/openssl:openssl +//third_party/zlib:zlibonly +//third_party/zlib:zlibonly_v1_2_3 +//third_party/python:headers +//third_party/openssl:crypto +``` + +### Aside: implicit dependencies + +The BUILD file for `//foo` never references +`//translations/tools:aggregator`. So, where's the direct dependency? + +Certain rules include implicit dependencies on additional libraries or tools. +For example, to build a `genproto` rule, you need first to build the Protocol +Compiler, so every `genproto` rule carries an implicit dependency on the +protocol compiler. These dependencies are not mentioned in the build file, +but added in by the build tool. The full set of implicit dependencies is + currently undocumented. Using `--noimplicit_deps` allows you to filter out + these deps from your query results. For cquery, this will include resolved toolchains. + +## Reverse dependencies + +You might want to know the set of targets that depends on some target. For instance, +if you're going to change some code, you might want to know what other code +you're about to break. You can use `rdeps(u, x)` to find the reverse +dependencies of the targets in `x` within the transitive closure of `u`. + +Bazel's [Sky Query](/query/language#sky-query) +supports the `allrdeps` function which allows you to query reverse dependencies +in a universe you specify. + +## Miscellaneous uses + +You can use `bazel query` to analyze many dependency relationships. + +### What exists ... + +#### What packages exist beneath `foo`? + +```bazel query 'foo/...' --output package``` + +#### What rules are defined in the `foo` package? + +```bazel query 'kind(rule, foo:*)' --output label_kind``` + +#### What files are generated by rules in the `foo` package? + +```bazel query 'kind("generated file", //foo:*)'``` + +#### What targets are generated by starlark macro `foo`? + +```bazel query 'attr(generator_function, foo, //path/to/search/...)'``` + +#### What's the set of BUILD files needed to build `//foo`? + +```bazel query 'buildfiles(deps(//foo))' | cut -f1 -d:``` + +#### What are the individual tests that a `test_suite` expands to? + +```bazel query 'tests(//foo:smoke_tests)'``` + +#### Which of those are C++ tests? + +```bazel query 'kind(cc_.*, tests(//foo:smoke_tests))'``` + +#### Which of those are small? Medium? Large? + +``` +bazel query 'attr(size, small, tests(//foo:smoke_tests))' + +bazel query 'attr(size, medium, tests(//foo:smoke_tests))' + +bazel query 'attr(size, large, tests(//foo:smoke_tests))' +``` + +#### What are the tests beneath `foo` that match a pattern? + +```bazel query 'filter("pa?t", kind(".*_test rule", //foo/...))'``` + +The pattern is a regex and is applied to the full name of the rule. It's similar to doing + +```bazel query 'kind(".*_test rule", //foo/...)' | grep -E 'pa?t'``` + +#### What package contains file `path/to/file/bar.java`? + +``` bazel query path/to/file/bar.java --output=package``` + +#### What is the build label for `path/to/file/bar.java?` + +```bazel query path/to/file/bar.java``` + +#### What rule target(s) contain file `path/to/file/bar.java` as a source? + +``` +fullname=$(bazel query path/to/file/bar.java) +bazel query "attr('srcs', $fullname, ${fullname//:*/}:*)" +``` + +### What package dependencies exist ... + +#### What packages does `foo` depend on? (What do I need to check out to build `foo`) + +```bazel query 'buildfiles(deps(//foo:foo))' --output package``` + +Note: `buildfiles` is required in order to correctly obtain all files +referenced by `subinclude`; see the reference manual for details. + +#### What packages does the `foo` tree depend on, excluding `foo/contrib`? + +```bazel query 'deps(foo/... except foo/contrib/...)' --output package``` + +### What rule dependencies exist ... + +#### What genproto rules does bar depend upon? + +```bazel query 'kind(genproto, deps(bar/...))'``` + +#### Find the definition of some JNI (C++) library that is transitively depended upon by a Java binary rule in the servlet tree. + +```bazel query 'some(kind(cc_.*library, deps(kind(java_binary, //java/com/example/frontend/...))))' --output location``` + +##### ...Now find the definitions of all the Java binaries that depend on them + +```bazel query 'let jbs = kind(java_binary, //java/com/example/frontend/...) in + let cls = kind(cc_.*library, deps($jbs)) in + $jbs intersect allpaths($jbs, $cls)' +``` + +### What file dependencies exist ... + +#### What's the complete set of Java source files required to build foo? + +Source files: + +```bazel query 'kind("source file", deps(//path/to/target/foo/...))' | grep java$``` + +Generated files: + +```bazel query 'kind("generated file", deps(//path/to/target/foo/...))' | grep java$``` + +#### What is the complete set of Java source files required to build QUX's tests? + +Source files: + +```bazel query 'kind("source file", deps(kind(".*_test rule", javatests/com/example/qux/...)))' | grep java$``` + +Generated files: + +```bazel query 'kind("generated file", deps(kind(".*_test rule", javatests/com/example/qux/...)))' | grep java$``` + +### What differences in dependencies between X and Y exist ... + +#### What targets does `//foo` depend on that `//foo:foolib` does not? + +```bazel query 'deps(//foo) except deps(//foo:foolib)'``` + +#### What C++ libraries do the `foo` tests depend on that the `//foo` production binary does _not_ depend on? + +```bazel query 'kind("cc_library", deps(kind(".*test rule", foo/...)) except deps(//foo))'``` + +### Why does this dependency exist ... + +#### Why does `bar` depend on `groups2`? + +```bazel query 'somepath(bar/...,groups2/...:*)'``` + +Once you have the results of this query, you will often find that a single +target stands out as being an unexpected or egregious and undesirable +dependency of `bar`. The query can then be further refined to: + +#### Show me a path from `docker/updater:updater_systest` (a `py_test`) to some `cc_library` that it depends upon: + +```bazel query 'let cc = kind(cc_library, deps(docker/updater:updater_systest)) in + somepath(docker/updater:updater_systest, $cc)'``` + +#### Why does library `//photos/frontend:lib` depend on two variants of the same library `//third_party/jpeglib` and `//third_party/jpeg`? + +This query boils down to: "show me the subgraph of `//photos/frontend:lib` that +depends on both libraries". When shown in topological order, the last element +of the result is the most likely culprit. + +```bazel query 'allpaths(//photos/frontend:lib, //third_party/jpeglib) + intersect + allpaths(//photos/frontend:lib, //third_party/jpeg)' +//photos/frontend:lib +//photos/frontend:lib_impl +//photos/frontend:lib_dispatcher +//photos/frontend:icons +//photos/frontend/modules/gadgets:gadget_icon +//photos/thumbnailer:thumbnail_lib +//third_party/jpeg/img:renderer +``` + +### What depends on ... + +#### What rules under bar depend on Y? + +```bazel query 'bar/... intersect allpaths(bar/..., Y)'``` + +Note: `X intersect allpaths(X, Y)` is the general idiom for the query "which X +depend on Y?" If expression X is non-trivial, it may be convenient to bind a +name to it using `let` to avoid duplication. + +#### What targets directly depend on T, in T's package? + +```bazel query 'same_pkg_direct_rdeps(T)'``` + +### How do I break a dependency ... + +{/* TODO find a convincing value of X to plug in here */} + +#### What dependency paths do I have to break to make `bar` no longer depend on X? + +To output the graph to a `svg` file: + +```bazel query 'allpaths(bar/...,X)' --output graph | dot -Tsvg > /tmp/dep.svg``` + +### Misc + +#### How many sequential steps are there in the `//foo-tests` build? + +Unfortunately, the query language can't currently give you the longest path +from x to y, but it can find the (or rather _a_) most distant node from the +starting point, or show you the _lengths_ of the longest path from x to every +y that it depends on. Use `maxrank`: + +```bazel query 'deps(//foo-tests)' --output maxrank | tail -1 +85 //third_party/zlib:zutil.c``` + +The result indicates that there exist paths of length 85 that must occur in +order in this build. diff --git a/reference/be/functions.mdx b/reference/be/functions.mdx deleted file mode 100644 index 4f18f06..0000000 --- a/reference/be/functions.mdx +++ /dev/null @@ -1,708 +0,0 @@ ---- -title: 'Functions' ---- - - - -## Contents - -- [package](#package) -- [package\_group](#package_group) -- [exports\_files](#exports_files) -- [glob](#glob) -- [select](#select) -- [subpackages](#subpackages) - -## package - -``` -package(default_deprecation, default_package_metadata, default_testonly, default_visibility, features) - -``` - -This function declares metadata that applies to every rule in the -package. It is used at most once within a package (BUILD file). - -For the counterpart that declares metadata applying to every rule in the whole -_repository_, use the `repo()` function in the -[`REPO.bazel` file](/external/overview#repo.bazel) at the root of your repo. -The `repo()` function takes exactly the same arguments as `package()`. - -The package() function should be called right after all the load() statements at the top of the -file, before any rule. - -### Arguments - -AttributeDescription`default_applicable_licenses` - -Alias for [`default_package_metadata`](#package.default_package_metadata). - - -`default_visibility` - -List of [labels](/concepts/labels); default is `[]` - -The default visibility of the top-level rule targets and symbolic -macros in this package — that is, the targets and symbolic macros -that are not themselves declared inside a symbolic macro. This attribute -is ignored if the target or macro specifies a `visibility` -value. - -For detailed information about the syntax of this attribute, see the -documentation of [visibility](/concepts/visibility). The package default visibility does not apply to -[exports\_files](#exports_files), which is public by default. - - -`default_deprecation` - -String; default is `""` - -Sets the default [`deprecation`](common-definitions.html#common.deprecation) message for all rules in this package. - -`default_package_metadata` - -List of [labels](/concepts/labels); default is `[]` - -Sets a default list of metadata targets which apply to all other targets in the package. -These are typically targets related to OSS package and license declarations. -See [rules\_license](https://github.com/bazelbuild/rules_license) for examples. - -`default_testonly` - -Boolean; default is `False` except as noted - -Sets the default [`testonly`](common-definitions.html#common.testonly) property for all rules in this package. - -In packages under `javatests` the default value is `True`. - -`features` - -List strings; default is `[]` - -Sets various flags that affect the semantics of this BUILD file. - -This feature is mainly used by the people working on the build system to -tag packages that need some kind of special handling. Do not use this unless -explicitly requested by someone working on the build system. - -### Examples - -The declaration below declares that the rules in this package are -visible only to members of package -group `//foo:target`. Individual visibility declarations -on a rule, if present, override this specification. - -``` -package(default_visibility = ["//foo:target"]) - -``` - -## package\_group - -``` -package_group(name, packages, includes) -``` - -This function defines a set of [packages](/concepts/build-ref#packages) -and associates a label with the set. The label can be referenced in -`visibility` attributes. - -Package groups are primarily used for visibility control. A publicly visible -target can be referenced from every package in the source tree. A privately -visible target can only be referenced within its own package (not subpackages). -In between these extremes, a target may allow access to its own package plus any -of the packages described by one or more package groups. For a more detailed -explanation of the visibility system, see the -[visibility](common-definitions.html#common.visibility) -attribute. - -A given package is considered to be in the group if it either matches the -`packages` attribute, or is already contained in one of the other -package groups mentioned in the `includes` attribute. - -Package groups are technically targets, but are not created by rules, and do -not themselves have any visibility protection. - -### Arguments - -AttributeDescription`name` - -[Name](/concepts/labels#target-names); required - -A unique name for this target. - -`packages` - -List of strings; default is `[]` - -A list of zero or more package specifications. - -Each package specification string can have one of the following -forms: - -1. The full name of a package, without its repository, starting with a - double slash. For example, `//foo/bar` specifies the package - having that name and which lives in the same repository as the package - group. - - -2. As above, but with a trailing `/...`. For example, ` - //foo/...` specifies the set of `//foo` and all its - subpackages. `//...` specifies all packages in the current - repository. - - -3. The strings `public` or `private`, which - respectively specify every package or no package. (This form requires - the flag `--incompatible_package_group_has_public_syntax` to - be set.) - - - -In addition, the first two kinds of package specifications may also -be prefixed with `-` to indicate that they are negated. - -The package group contains any package that matches at least one of -its positive specifications and none of its negative specifications -For instance, the value `[//foo/..., -//foo/tests/...]` -includes all subpackages of `//foo` that are not also -subpackages of `//foo/tests`. ( `//foo` itself is -included while //foo/tests itself is not.) - -Aside from public visibility, there is no way to directly specify -packages outside the current repository. - -If this attribute is missing, it is the same as setting it to an -empty list, which is also the same as setting it to a list containing -only `private`. - - - -_Note:_ Prior to Bazel 6.0, the specification `//...` -had a legacy behavior of being the same as `public`. This -behavior is fixed when -`--incompatible_fix_package_group_reporoot_syntax` is -enabled, which is the default after Bazel 6.0. - -_Note:_ Prior to Bazel 6.0, when this attribute is serialized as -part of `bazel query --output=proto` (or -`--output=xml`), the leading slashes are omitted. For -instance, `//pkg/foo/...` will output as -`\"pkg/foo/...\"`. This behavior is fixed when -`--incompatible_package_group_includes_double_slash` is -enabled, which is the default after Bazel 6.0. - -`includes` - -List of [labels](/concepts/labels); default is `[]` - -Other package groups that are included in this one. - -The labels in this attribute must refer to other package groups. -Packages in referenced package groups are taken to be part of this -package group. This is transitive — if package group -`a` includes package group `b`, and `b` -includes package group `c`, then every package in -`c` will also be a member of `a`. - -When used together with negated package specifications, note that the -set of packages for each group is first computed independently and the -results are then unioned together. This means that negated -specifications in one group have no effect on the specifications in -another group. - -### Examples - -The following `package_group` declaration specifies a -package group called "tropical" that contains tropical fruits. - -``` -package_group( - name = "tropical", - packages = [ - "//fruits/mango", - "//fruits/orange", - "//fruits/papaya/...", - ], -) - -``` - -The following declarations specify the package groups of a fictional -application: - -``` -package_group( - name = "fooapp", - includes = [ - ":controller", - ":model", - ":view", - ], -) - -package_group( - name = "model", - packages = ["//fooapp/database"], -) - -package_group( - name = "view", - packages = [ - "//fooapp/swingui", - "//fooapp/webui", - ], -) - -package_group( - name = "controller", - packages = ["//fooapp/algorithm"], -) - -``` - -## exports\_files - -``` -exports_files([label, ...], visibility, licenses) -``` - -`exports_files()` specifies a list of files belonging to -this package that are exported to other packages. - -The BUILD file for a package may only refer directly to source files belonging -to another package if they are explicitly exported with an -`exports_files()` statement. Read more about -[visibility of files](/concepts/visibility#source-file-target-visibility). - -As a legacy behaviour, also files mentioned as input to a rule are exported -with the default visibility until the flag -[`--incompatible_no_implicit_file_export`](https://github.com/bazelbuild/bazel/issues/10225) -is flipped. However, this behavior should not be relied upon and actively -migrated away from. - -### Arguments - -The argument is a list of names of files within the current package. A -visibility declaration can also be specified; in this case, the files will be -visible to the targets specified. If no visibility is specified, the files -will be visible to every package, even if a package default visibility was -specified in the `package` -function. The [licenses](common-definitions.html#common.licenses) -can also be specified. - -### Example - -The following example exports `golden.txt`, a -text file from the `test_data` package, so that other -packages may use it, for example, in the `data` attribute -of tests. - -``` -# from //test_data/BUILD - -exports_files(["golden.txt"]) - -``` - -## glob - -``` -glob(include, exclude=[], exclude_directories=1, allow_empty=True) -``` - -Glob is a helper function that finds all files that match certain path patterns, -and returns a new, mutable, sorted list of their paths. Glob only searches files -in its own package, and looks only for source files (not generated files nor -other targets). - -A source file's Label is included in the result if the file's package-relative -path matches any of the `include` patterns and none of the -`exclude` patterns. - -The `include` and `exclude` lists contain path patterns -that are relative to the current package. Every pattern may consist of one or -more path segments. As usual with Unix paths, these segments are separated by -`/`. The segments in the pattern are matched against the segments of -the path. Segments may contain the `*` wildcard: this matches -any substring in the path segment (even the empty substring), excluding the -directory separator `/`. This wildcard can be used multiple times -within one path segment. Additionally, the `**` wildcard can match -zero or more complete path segments, but it must be declared as a standalone -path segment. - -Examples: - -- `foo/bar.txt` matches exactly the `foo/bar.txt` file - in this package (unless `foo/` is a subpackage) -- `foo/*.txt` matches every file in the `foo/` directory - if the file ends with `.txt` (unless `foo/` is a - subpackage) -- `foo/a*.htm*` matches every file in the `foo/` - directory that starts with `a`, then has an arbitrary string (could - be empty), then has `.htm`, and ends with another arbitrary string - (unless `foo/` is a subpackage); such as `foo/axx.htm` - and `foo/a.html` or `foo/axxx.html` -- `foo/*` matches every file in the `foo/` directory, - (unless `foo/` is a subpackage); it does not match `foo` - directory itself even if `exclude_directories` is set to - 0 -- `foo/**` matches every file in every non-subpackage subdirectory - under package's first level subdirectory `foo/`; if - `exclude_directories` is set to 0, `foo` - directory itself also matches the pattern; in this case, `**` is - considered to match zero path segments -- `**/a.txt` matches `a.txt` files in this package's - directory plus non-subpackage subdirectories. -- `**/bar/**/*.txt` matches every `.txt` file in every - non-subpackage subdirectory of this package, if at least one directory on the - resulting path is called `bar`, such as - `xxx/bar/yyy/zzz/a.txt` or `bar/a.txt` (remember that - `**` also matches zero segments) or `bar/zzz/a.txt` -- `**` matches every file in every non-subpackage subdirectory of - this package -- `foo**/a.txt` is an invalid pattern, because `**` must - stand on its own as a segment -- `foo/` is an invalid pattern, because the second segment defined - after `/` is an empty string - -If the `exclude_directories` argument is enabled (set to 1), files of -type directory will be omitted from the results (default 1). - -If the `allow_empty` argument is set to `False`, the -`glob` function will error-out if the result would otherwise be the -empty list. - -There are several important limitations and caveats: - -1. Since `glob()` runs during BUILD file evaluation, - `glob()` matches files only in your source tree, never - generated files. If you are building a target that requires both - source and generated files, you must append an explicit list of generated - files to the glob. See the [example](#glob_example) - below with `:mylib` and `:gen_java_srcs`. - - -2. If a rule has the same name as a matched source file, the rule will - "shadow" the file. - - - - To understand this, remember that `glob()` returns a list of - paths, so using `glob()` in other rules' attribute (e.g. - `srcs = glob(["*.cc"])`) has the same effect as listing the - matched paths explicitly. If for example `glob()` yields - `["Foo.java", "bar/Baz.java"]` but there's also a rule in the - package called "Foo.java" (which is allowed, though Bazel warns about it), - then the consumer of the `glob()` will use the "Foo.java" rule - (its outputs) instead of the "Foo.java" file. See - [GitHub\ - issue #10395](https://github.com/bazelbuild/bazel/issues/10395#issuecomment-583714657) for more details. - - -3. Globs may match files in subdirectories. And subdirectory names - may be wildcarded. However... - -4. Labels are not allowed to cross the package boundary and glob does - not match files in subpackages. - - - - For example, the glob expression `**/*.cc` in package - `x` does not include `x/y/z.cc` if - `x/y` exists as a package (either as - `x/y/BUILD`, or somewhere else on the package-path). This - means that the result of the glob expression actually depends on the - existence of BUILD files — that is, the same glob expression would - include `x/y/z.cc` if there was no package called - `x/y` or it was marked as deleted using the - [--deleted\_packages](/docs/user-manual#flag--deleted_packages) - flag. - - -5. The restriction above applies to all glob expressions, - no matter which wildcards they use. - -6. A hidden file with filename starting with `.` is completely matched by - both the `**` and the `*` wildcards. If you want to match a hidden file - with a compound pattern, your pattern needs to begin with a `.`. For example, - `*` and `.*.txt` will match `.foo.txt`, but `*.txt` - will not. - - Hidden directories are also matched in the same manner. Hidden directories - may include files that are not required as inputs, and can increase the - number of unnecessarily globbed files and memory consumption. To exclude - hidden directories, add them to the "exclude" list argument. - -7. The "\*\*" wildcard has one corner case: the pattern - `"**"` doesn't match the package's directory path. That is to - say, `glob(["**"], exclude_directories = False)` matches all files - and directories transitively strictly under the current package's directory - (but of course not going into directories of subpackages - see the previous - note about that). - - -In general, you should **try to provide an appropriate extension (e.g. \*.html)** -**instead of using a bare '\*'** for a glob pattern. The more explicit name -is both self documenting and ensures that you don't accidentally match backup -files, or emacs/vi/... auto-save files. - -When writing build rules you can enumerate the elements of the glob. This -enables generating individual rules for every input, for example. See the -[expanded glob example](#expanded_glob_example) section below. - -### Glob Examples - -Create a Java library built from all java files in this directory, -and all files generated by the `:gen_java_srcs` rule. - -``` -java_library( - name = "mylib", - srcs = glob(["*.java"]) + [":gen_java_srcs"], - deps = "...", -) - -genrule( - name = "gen_java_srcs", - outs = [ - "Foo.java", - "Bar.java", - ], - ... -) - -``` - -Include all txt files in directory testdata except experimental.txt. -Note that files in subdirectories of testdata will not be included. If -you want those files to be included, use a recursive glob (\*\*). - -``` -sh_test( - name = "mytest", - srcs = ["mytest.sh"], - data = glob( - ["testdata/*.txt"], - exclude = ["testdata/experimental.txt"], - ), -) - -``` - -### Recursive Glob Examples - -Make the test depend on all txt files in the testdata directory and any -of its subdirectories (and their subdirectories, and so on). -Subdirectories containing a BUILD file are ignored. (See limitations -and caveats above.) - -``` -sh_test( - name = "mytest", - srcs = ["mytest.sh"], - data = glob(["testdata/**/*.txt"]), -) - -``` - -Create a library built from all java files in this directory and all -subdirectories except those whose path includes a directory named testing. -**This pattern should be avoided if possible, as it can reduce build** -**incrementality and therefore increase build times.** - -``` -java_library( - name = "mylib", - srcs = glob( - ["**/*.java"], - exclude = ["**/testing/**"], - ), -) - -``` - -### Expanded Glob Examples - -Create an individual genrule for \*\_test.cc in the current directory -that counts the number of lines in the file. - -``` -# Conveniently, the build language supports list comprehensions. -[genrule( - name = "count_lines_" + f[:-3], # strip ".cc" - srcs = [f], - outs = ["%s-linecount.txt" % f[:-3]], - cmd = "wc -l $< >$@", - ) for f in glob(["*_test.cc"])] - -``` - -If the BUILD file above is in package //foo and the package contains three -matching files, a\_test.cc, b\_test.cc and c\_test.cc then running -`bazel query '//foo:all'` will list all rules that were generated: - -``` -$ bazel query '//foo:all' | sort -//foo:count_lines_a_test -//foo:count_lines_b_test -//foo:count_lines_c_test - -``` - -## select - -``` -select( - {conditionA: valuesA, conditionB: valuesB, ...}, - no_match_error = "custom message" -) - -``` - -`select()` is the helper function that makes a rule attribute -[configurable](common-definitions.html#configurable-attributes). -It can replace the right-hand side of - -_almost_ -any attribute assignment so its value depends on command-line Bazel flags. -You can use this, for example, to define platform-specific dependencies or to -embed different resources depending on whether a rule is built in "developer" -vs. "release" mode. - -Basic use is as follows: - -``` -sh_binary( - name = "mytarget", - srcs = select({ - ":conditionA": ["mytarget_a.sh"], - ":conditionB": ["mytarget_b.sh"], - "//conditions:default": ["mytarget_default.sh"] - }) -) - -``` - -This makes the `srcs` attribute of -a `sh_binary` configurable by replacing its normal label -list assignment with a `select` call that maps -configuration conditions to matching values. Each condition is a label -reference to -a `config_setting` or -`constraint_value`, -which "matches" if the target's configuration matches an expected set of -values. The value of `mytarget#srcs` then becomes whichever -label list matches the current invocation. - -Notes: - -- Exactly one condition is selected on any invocation. - -- If multiple conditions match and one is a specialization of the others, - the specialization takes precedence. Condition B is considered a - specialization of condition A if B has all the same flags and constraint - values as A plus some additional flags or constraint values. This also - means that specialization resolution is not designed to create an ordering as - demonstrated in Example 2 below. - -- If multiple conditions match and one is not a specialization of all the - others, Bazel fails with an error, unless all conditions resolve to the same value. - -- The special pseudo-label `//conditions:default` is - considered to match if no other condition matches. If this condition - is left out, some other rule must match to avoid an error. - -- `select` can be embedded _inside_ a larger - attribute assignment. So `srcs = ["common.sh"] - + select({ ":conditionA": ["myrule_a.sh"], ...})` and ` - srcs = select({ ":conditionA": ["a.sh"]}) + select({ ":conditionB": - ["b.sh"]})` are valid expressions. - -- `select` works with most, but not all, attributes. Incompatible - attributes are marked `nonconfigurable` in their documentation. - - - - ## subpackages - - - - ``` - subpackages(include, exclude=[], allow_empty=True) - ``` - - - `subpackages()` is a helper function, similar to `glob()` - that lists subpackages instead of files and directories. It uses the same - path patterns as `glob()` and can match any subpackage that is a - direct descendant of the currently loading BUILD file. See [glob](#glob) for a detailed explanation and examples of include and - exclude patterns. - - - - The resulting list of subpackages returned is in sorted order and contains - paths relative to the current loading package that match the given patterns in - `include` and not those in `exclude`. - - - - - ### Example - - - - The following example lists all the direct subpackages for the package `foo/BUILD` - - - ``` - # The following BUILD files exist: - # foo/BUILD - # foo/bar/baz/BUILD - # foo/bar/but/bad/BUILD - # foo/sub/BUILD - # foo/sub/deeper/BUILD - # - # In foo/BUILD a call to - subs1 = subpackages(include = ["**"]) - - # results in subs1 == ["sub", "bar/baz", "bar/but/bad"] - # - # 'sub/deeper' is not included because it is a subpackage of 'foo/sub' not of - # 'foo' - - subs2 = subpackages(include = ["bar/*"]) - # results in subs2 = ["bar/baz"] - # - # Since 'bar' is not a subpackage itself, this looks for any subpackages under - # all first level subdirectories of 'bar'. - - subs3 = subpackages(include = ["bar/**"]) - # results in subs3 = ["bar/baz", "bar/but/bad"] - # - # Since bar is not a subpackage itself, this looks for any subpackages which are - # (1) under all subdirectories of 'bar' which can be at any level, (2) not a - # subpackage of another subpackages. - - subs4 = subpackages(include = ["sub"]) - subs5 = subpackages(include = ["sub/*"]) - subs6 = subpackages(include = ["sub/**"]) - # results in subs4 and subs6 being ["sub"] - # results in subs5 = []. - # - # In subs4, expression "sub" checks whether 'foo/sub' is a package (i.e. is a - # subpackage of 'foo'). - # In subs5, "sub/*" looks for subpackages under directory 'foo/sub'. Since - # 'foo/sub' is already a subpackage itself, the subdirectories will not be - # traversed anymore. - # In subs6, 'foo/sub' is a subpackage itself and matches pattern "sub/**", so it - # is returned. But the subdirectories of 'foo/sub' will not be traversed - # anymore. - - ``` - - - - In general it is preferred that instead of calling this function directly - that users use the 'subpackages' module of - [skylib](https://github.com/bazelbuild/bazel-skylib). diff --git a/reference/be/general.mdx b/reference/be/general.mdx index d96013c..8c6d64d 100644 --- a/reference/be/general.mdx +++ b/reference/be/general.mdx @@ -982,8 +982,11 @@ in the Bazel source tree. - `name.textproto` (only built if explicitly requested): the text proto version of `name.binaryproto`. -Warning: the output format of this rule is not guaranteed to be stable. It is intended mainly for -internal use by [Stardoc](https://github.com/bazelbuild/stardoc). +Note: the exact output of this rule is not a stable public API. For example, the set of +natively-defined common rule attributes and their docstrings may change even with minor Bazel +releases. For this reason, documentation generated for user-defined rules is not stable across Bazel +releases, so we suggest taking care that any "golden tests" based on outputs of this rule are only +run with a single Bazel version. diff --git a/reference/be/platforms-and-toolchains.mdx b/reference/be/platforms-and-toolchains.mdx deleted file mode 100644 index 1844b00..0000000 --- a/reference/be/platforms-and-toolchains.mdx +++ /dev/null @@ -1,544 +0,0 @@ ---- -title: 'Platforms and Toolchains Rules' ---- - - - -This set of rules exists to allow you to model specific hardware platforms you are -building for and specify the specific tools you may need to compile code for those platforms. -The user should be familiar with the concepts explained [here](/extending/platforms). - -## Rules - -- [constraint\_setting](#constraint_setting) -- [constraint\_value](#constraint_value) -- [platform](#platform) -- [toolchain](#toolchain) -- [toolchain\_type](#toolchain_type) - -## constraint\_setting - -[View rule sourceopen\_in\_new](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/platform/ConstraintSettingRule.java) - -``` -constraint_setting(name, aspect_hints, default_constraint_value, deprecation, features, licenses, tags, testonly, visibility) -``` - -This rule is used to introduce a new constraint type for which a platform may specify a value. -For instance, you might define a `constraint_setting` named "glibc\_version" to represent -the capability for platforms to have different versions of the glibc library installed. - -For more details, see the -[Platforms](https://bazel.build/docs/platforms) page. - -Each `constraint_setting` has an extensible set of associated -`constraint_value` s. Usually these are defined in the same package, but sometimes a -different package will introduce new values for an existing setting. For instance, the predefined -setting `@platforms//cpu:cpu` can be extended with a custom value in order to -define a platform targeting an obscure cpu architecture. - - - -### Arguments - -Attributes`name` - -[Name](/concepts/labels#target-names); required - -A unique name for this target. - -`default_constraint_value` - -[Name](/concepts/labels#target-names); [nonconfigurable](common-definitions.html#configurable-attributes); default is `None` - - The label of the default value for this setting, to be used if no value is given. If this - attribute is present, the `constraint_value` it points to must be defined in the - same package as this `constraint_setting`. - - - -If a constraint setting has a default value, then whenever a platform does not include -any constraint value for that setting, it is the same as if the platform had specified the -default value. Otherwise, if there is no default value, the constraint setting is considered -to be unspecified by that platform. In that case, the platform would not match against any -constraint list (such as for a `config_setting`) that requires a particular value -for that setting. - - - -## constraint\_value - -[View rule sourceopen\_in\_new](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/platform/ConstraintValueRule.java) - -``` -constraint_value(name, aspect_hints, constraint_setting, deprecation, features, licenses, tags, testonly, visibility) -``` - -This rule introduces a new value for a given constraint type. - -For more details, see the -[Platforms](https://bazel.build/docs/platforms) page. - -#### Example - -The following creates a new possible value for the predefined `constraint_value` -representing cpu architecture. - -``` -constraint_value( - name = "mips", - constraint_setting = "@platforms//cpu:cpu", -) - -``` - -Platforms can then declare that they have the `mips` architecture as an alternative to -`x86_64`, `arm`, and so on. - - - -### Arguments - -Attributes`name` - -[Name](/concepts/labels#target-names); required - -A unique name for this target. - -`constraint_setting` - -[Label](/concepts/labels); [nonconfigurable](common-definitions.html#configurable-attributes); required - - The `constraint_setting` for which this `constraint_value` is a - possible choice. - - - -## platform - -[View rule sourceopen\_in\_new](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/platform/PlatformRule.java) - -``` -platform(name, aspect_hints, constraint_values, deprecation, exec_properties, features, flags, licenses, missing_toolchain_error, parents, remote_execution_properties, required_settings, tags, testonly, visibility) -``` - -This rule defines a new platform -- a named collection of constraint choices -(such as cpu architecture or compiler version) describing an environment in -which part of the build may run. - -For more details, see the [Platforms](/extending/platforms) page. - -#### Example - -This defines a platform that describes any environment running Linux on ARM. - -``` -platform( - name = "linux_arm", - constraint_values = [ - "@platforms//os:linux", - "@platforms//cpu:arm", - ], -) - -``` - -### Platform Flags - -Platforms may use the `flags` attribute to specify a list of flags that will be added -to the configuration whenever the platform is used as the target platform (i.e., as the value of -the `--platforms` flag). - -Flags set from the platform effectively have the highest precedence and overwrite any previous -value for that flag, from the command line, rc file, or transition. - -#### Example - -``` -platform( - name = "foo", - flags = [ - "--dynamic_mode=fully", - "--//bool_flag", - "--no//package:other_bool_flag", - ], -) - -``` - -This defines a platform named `foo`. When this is the target platform (either because -the user specified `--platforms//:foo`, because a transition set the -`//command_line_option:platforms` flag to `["//:foo"]`, or because -`//:foo` was used as an execution platform), then the given flags will be set in the -configuration. - -#### Platforms and Repeatable Flags - -Some flags will accumulate values when they are repeated, such as `--features`, -`--copt`, any Starlark flag created as `config.string(repeatable = True)`. -These flags are not compatible with setting the flags from the platform: instead, all previous -values will be removed and overwritten with the values from the platform. - -As an example, given the following platform, the invocation `build --platforms=//:repeat_demo - --features feature_a --features feature_b` will end up with the value of the -`--feature` flag being `["feature_c", "feature_d"]`, removing the features -set on the command line. - -``` -platform( - name = "repeat_demo", - flags = [ - "--features=feature_c", - "--features=feature_d", - ], -) - -``` - -For this reason, it is discouraged to use repeatable flags in the `flags` attribute. - -### Platform Inheritance - -Platforms may use the `parents` attribute to specify another platform that they will -inherit constraint values from. Although the `parents` attribute takes a list, no -more than a single value is currently supported, and specifying multiple parents is an error. - -When checking for the value of a constraint setting in a platform, first the values directly set -(via the `constraint_values` attribute) are checked, and then the constraint values on -the parent. This continues recursively up the chain of parent platforms. In this manner, any -values set directly on a platform will override the values set on the parent. - -Platforms inherit the `exec_properties` attribute from the parent platform. -The dictionary entries in `exec_properties` of the parent and child platforms -will be combined. -If the same key appears in both the parent's and the child's `exec_properties`, -the child's value will be used. If the child platform specifies an empty string as a value, the -corresponding property will be unset. - -Platforms can also inherit the (deprecated) `remote_execution_properties` attribute -from the parent platform. Note: new code should use `exec_properties` instead. The -logic described below is maintained to be compatible with legacy behavior but will be removed -in the future. - -The logic for setting the `remote_execution_platform` is as follows when there -is a parent platform: - - - -1. If `remote_execution_property` is not set on the child platform, the parent's - `remote_execution_properties` will be used. - -2. If `remote_execution_property` is set on the child platform, and contains the - literal string {PARENT\_REMOTE\_EXECUTION\_PROPERTIES}, that macro will be - replaced with the contents of the parent's `remote_execution_property` attribute. - -3. If `remote_execution_property` is set on the child platform, and does not contain - the macro, the child's `remote_execution_property` will be used unchanged. - - -_Since `remote_execution_properties` is deprecated and will be phased out, mixing_ -_`remote_execution_properties` and `exec_properties` in the same_ -_inheritance chain is not allowed._ -Prefer to use `exec_properties` over the deprecated -`remote_execution_properties`. - -#### Example: Constraint Values - -``` -platform( - name = "parent", - constraint_values = [ - "@platforms//os:linux", - "@platforms//cpu:arm", - ], -) -platform( - name = "child_a", - parents = [":parent"], - constraint_values = [ - "@platforms//cpu:x86_64", - ], -) -platform( - name = "child_b", - parents = [":parent"], -) - -``` - -In this example, the child platforms have the following properties: - - - -- `child_a` has the constraint values `@platforms//os:linux` (inherited - from the parent) and `@platforms//cpu:x86_64` (set directly on the platform). - -- `child_b` inherits all constraint values from the parent, and doesn't set any of - its own. - - -#### Example: Execution properties - -``` -platform( - name = "parent", - exec_properties = { - "k1": "v1", - "k2": "v2", - }, -) -platform( - name = "child_a", - parents = [":parent"], -) -platform( - name = "child_b", - parents = [":parent"], - exec_properties = { - "k1": "child" - } -) -platform( - name = "child_c", - parents = [":parent"], - exec_properties = { - "k1": "" - } -) -platform( - name = "child_d", - parents = [":parent"], - exec_properties = { - "k3": "v3" - } -) - -``` - -In this example, the child platforms have the following properties: - - - -- `child_a` inherits the "exec\_properties" of the parent and does not set its own. - -- `child_b` inherits the parent's `exec_properties` and overrides the - value of `k1`. Its `exec_properties` will be: - `{ "k1": "child", "k2": "v2" }`. - -- `child_c` inherits the parent's `exec_properties` and unsets - `k1`. Its `exec_properties` will be: - `{ "k2": "v2" }`. - -- `child_d` inherits the parent's `exec_properties` and adds a new - property. Its `exec_properties` will be: - `{ "k1": "v1", "k2": "v2", "k3": "v3" }`. - - -### Arguments - -Attributes`name` - -[Name](/concepts/labels#target-names); required - -A unique name for this target. - -`constraint_values` - -List of [labels](/concepts/labels); [nonconfigurable](common-definitions.html#configurable-attributes); default is `[]` - - The combination of constraint choices that this platform comprises. In order for a platform - to apply to a given environment, the environment must have at least the values in this list. - - - -Each `constraint_value` in this list must be for a different -`constraint_setting`. For example, you cannot define a platform that requires the -cpu architecture to be both `@platforms//cpu:x86_64` and -`@platforms//cpu:arm`. - - - -`exec_properties` - -Dictionary: String -> String; [nonconfigurable](common-definitions.html#configurable-attributes); default is `{}` - - A map of strings that affect the way actions are executed remotely. Bazel makes no attempt - to interpret this, it is treated as opaque data that's forwarded via the Platform field of - the [remote execution protocol](https://github.com/bazelbuild/remote-apis). - - This includes any data from the parent platform's `exec_properties` attributes. - If the child and parent platform define the same keys, the child's values are kept. Any - keys associated with a value that is an empty string are removed from the dictionary. - - This attribute is a full replacement for the deprecated - `remote_execution_properties`. - - `flags` - -List of strings; [nonconfigurable](common-definitions.html#configurable-attributes); default is `[]` - - A list of flags that will be enabled when this platform is used as the target platform in - a configuration. Only flags that are part of the configuration can be set, such as those - that can be used in transitions, or the `--define` flag. - - `missing_toolchain_error` - -String; [nonconfigurable](common-definitions.html#configurable-attributes); default is `"For more information on platforms or toolchains see https://bazel.build/concepts/platforms-intro."` - - A custom error message that is displayed when a mandatory toolchain requirement cannot be satisfied for this target platform. Intended to point to relevant documentation users can read to understand why their toolchains are misconfigured. - - Not inherited from parent platforms. - - `parents` - -List of [labels](/concepts/labels); [nonconfigurable](common-definitions.html#configurable-attributes); default is `[]` - - The label of a `platform` target that this platform should inherit from. Although - the attribute takes a list, there should be no more than one platform present. Any - constraint\_settings not set directly on this platform will be found in the parent platform. - See the section on [Platform Inheritance](#platform_inheritance) for details. - - `remote_execution_properties` - -String; [nonconfigurable](common-definitions.html#configurable-attributes); default is `""` - - DEPRECATED. Please use exec\_properties attribute instead. - - A string used to configure a remote execution platform. Actual builds make no attempt to - interpret this, it is treated as opaque data that can be used by a specific SpawnRunner. - This can include data from the parent platform's "remote\_execution\_properties" attribute, - by using the macro "{PARENT\_REMOTE\_EXECUTION\_PROPERTIES}". See the section on - [Platform Inheritance](#platform_inheritance) for details. - - `required_settings` - -List of [labels](/concepts/labels); default is `[]` - - A list of `config_setting` s that must be satisfied by the target configuration - in order for this platform to be used as an execution platform during toolchain resolution. - - Required settings are not inherited from parent platforms. - - - -## toolchain - -[View rule sourceopen\_in\_new](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/platform/ToolchainRule.java) - -``` -toolchain(name, aspect_hints, deprecation, exec_compatible_with, features, licenses, package_metadata, tags, target_compatible_with, target_settings, testonly, toolchain, toolchain_type, use_target_platform_constraints, visibility) -``` - -This rule declares a specific toolchain's type and constraints so that it can be selected -during toolchain resolution. See the -[Toolchains](https://bazel.build/docs/toolchains) page for more -details. - - - -### Arguments - -Attributes`name` - -[Name](/concepts/labels#target-names); required - -A unique name for this target. - -`exec_compatible_with` - -List of [labels](/concepts/labels); [nonconfigurable](common-definitions.html#configurable-attributes); default is `[]` - - A list of `constraint_value` s that must be satisfied by an execution platform in - order for this toolchain to be selected for a target building on that platform. - - `target_compatible_with` - -List of [labels](/concepts/labels); [nonconfigurable](common-definitions.html#configurable-attributes); default is `[]` - - A list of `constraint_value` s that must be satisfied by the target platform in - order for this toolchain to be selected for a target building for that platform. - - `target_settings` - -List of [labels](/concepts/labels); default is `[]` - - A list of `config_setting` s that must be satisfied by the target configuration - in order for this toolchain to be selected during toolchain resolution. - - `toolchain` - -[Name](/concepts/labels#target-names); required - - The target representing the actual tool or tool suite that is made available when this - toolchain is selected. - - `toolchain_type` - -[Label](/concepts/labels); [nonconfigurable](common-definitions.html#configurable-attributes); required - - The label of a `toolchain_type` target that represents the role that this - toolchain serves. - - `use_target_platform_constraints` - -Boolean; [nonconfigurable](common-definitions.html#configurable-attributes); default is `False` - - If `True`, this toolchain behaves as if its `exec_compatible_with` and - `target_compatible_with` constraints are set to those of the current target - platform. `exec_compatible_with` and `target_compatible_with` must not - be set in that case. - - - -## toolchain\_type - -[View rule sourceopen\_in\_new](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/ToolchainType.java) - -``` -toolchain_type(name, aspect_hints, compatible_with, deprecation, features, no_match_error, package_metadata, restricted_to, tags, target_compatible_with, testonly, visibility) -``` - -This rule defines a new type of toolchain -- a simple target that represents a class of tools that -serve the same role for different platforms. - -See the [Toolchains](/docs/toolchains) page for more details. - -#### Example - -This defines a toolchain type for a custom rule. - -``` -toolchain_type( - name = "bar_toolchain_type", -) - -``` - -This can be used in a bzl file. - -``` -bar_binary = rule( - implementation = _bar_binary_impl, - attrs = { - "srcs": attr.label_list(allow_files = True), - ... - # No `_compiler` attribute anymore. - }, - toolchains = ["//bar_tools:toolchain_type"] -) - -``` - -### Arguments - -Attributes`name` - -[Name](/concepts/labels#target-names); required - -A unique name for this target. - -`no_match_error` - -String; [nonconfigurable](common-definitions.html#configurable-attributes); default is `""` - - A custom error message to display when no matching toolchain is found for this type. diff --git a/reference/be/protocol-buffer.mdx b/reference/be/protocol-buffer.mdx index 61bb4f1..93ad701 100644 --- a/reference/be/protocol-buffer.mdx +++ b/reference/be/protocol-buffer.mdx @@ -166,7 +166,7 @@ rules to generate Java code for. [View rule sourceopen\_in\_new](https://github.com/protocolbuffers/protobuf/tree/v29.0-rc2/bazel/private/proto_library_rule.bzl) ``` -proto_library(name, deps, srcs, data, allow_exports, aspect_hints, compatible_with, deprecation, exec_compatible_with, exec_group_compatible_with, exec_properties, exports, features, import_prefix, licenses, option_deps, package_metadata, restricted_to, strip_import_prefix, tags, target_compatible_with, testonly, toolchains, visibility) +proto_library(name, deps, srcs, data, allow_exports, aspect_hints, compatible_with, deprecation, exec_compatible_with, exec_group_compatible_with, exec_properties, exports, extension_declarations, features, import_prefix, licenses, option_deps, package_metadata, restricted_to, strip_import_prefix, tags, target_compatible_with, testonly, toolchains, visibility) ``` Use `proto_library` to define libraries of protocol buffers which @@ -238,6 +238,12 @@ It's an error if you use "import public" but do not list the corresponding libra in the exports attribute. Note that you have list the library both in deps and exports since not all lang\_proto\_library implementations have been changed yet. + `extension_declarations` + +List of [labels](/concepts/labels); default is `[]` + + List of files containing extension declarations. This attribute is only allowed +for use with MessageSet. `import_prefix` String; default is `""` diff --git a/reference/be/python.mdx b/reference/be/python.mdx index 32acec6..ec9e019 100644 --- a/reference/be/python.mdx +++ b/reference/be/python.mdx @@ -45,7 +45,7 @@ The order of this list can matter because it affects the order that information from dependencies is merged in, which can be relevant depending on the ordering mode of depsets that are merged. -\\* {obj}\`PyInfo.site\_packages\_symlinks\` uses topological ordering. +\\* {obj}\`PyInfo.venv\_symlinks\` uses default ordering. See {obj}\`PyInfo\` for more information about the ordering of its depsets and how its fields are merged. @@ -333,7 +333,7 @@ The order of this list can matter because it affects the order that information from dependencies is merged in, which can be relevant depending on the ordering mode of depsets that are merged. -\\* {obj}\`PyInfo.site\_packages\_symlinks\` uses topological ordering. +\\* {obj}\`PyInfo.venv\_symlinks\` uses default ordering. See {obj}\`PyInfo\` for more information about the ordering of its depsets and how its fields are merged. @@ -385,19 +385,21 @@ under the binary's venv site-packages directory that should be made available (i namespace packages\]( https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#native-namespace-packages). However, the \*content\* of the files cannot be taken into account, merely their -presence or absense. Stated another way: \[pkgutil-style namespace packages\]( +presence or absence. Stated another way: \[pkgutil-style namespace packages\]( https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages) won't be understood as namespace packages; they'll be seen as regular packages. This will likely lead to conflicts with other targets that contribute to the namespace. -:::{tip} -This attributes populates {obj}\`PyInfo.site\_packages\_symlinks\`, which is -a topologically ordered depset. This means dependencies closer and earlier -to a consumer have precedence. See {obj}\`PyInfo.site\_packages\_symlinks\` for -more information. +:::{seealso} +This attributes populates {obj}\`PyInfo.venv\_symlinks\`. ::: :::{versionadded} 1.4.0 +::: +:::{versionchanged} 1.5.0 +The topological order has been removed and if 2 different versions of the same PyPI +package are observed, the behaviour has no guarantees except that it is deterministic +and that only one package version will be included. ::: `imports` @@ -535,7 +537,7 @@ The order of this list can matter because it affects the order that information from dependencies is merged in, which can be relevant depending on the ordering mode of depsets that are merged. -\\* {obj}\`PyInfo.site\_packages\_symlinks\` uses topological ordering. +\\* {obj}\`PyInfo.venv\_symlinks\` uses default ordering. See {obj}\`PyInfo\` for more information about the ordering of its depsets and how its fields are merged. @@ -778,7 +780,7 @@ be avoided if possible. [View rule sourceopen\_in\_new](https://github.com/bazelbuild/rules_python/tree/0.40.0/python/private/py_runtime_rule.bzl) ``` -py_runtime(name, abi_flags, aspect_hints, bootstrap_template, compatible_with, coverage_tool, deprecation, exec_compatible_with, exec_group_compatible_with, exec_properties, features, files, implementation_name, interpreter, interpreter_path, interpreter_version_info, package_metadata, pyc_tag, python_version, restricted_to, site_init_template, stage2_bootstrap_template, stub_shebang, tags, target_compatible_with, testonly, toolchains, visibility, zip_main_template) +py_runtime(name, abi_flags, aspect_hints, bootstrap_template, compatible_with, coverage_tool, deprecation, exec_compatible_with, exec_group_compatible_with, exec_properties, features, files, implementation_name, interpreter, interpreter_path, interpreter_version_info, package_metadata, pyc_tag, python_version, restricted_to, site_init_template, stage2_bootstrap_template, stub_shebang, supports_build_time_venv, tags, target_compatible_with, testonly, toolchains, visibility, zip_main_template) ``` Represents a Python runtime used to execute Python code. @@ -971,6 +973,16 @@ See https://github.com/bazelbuild/bazel/issues/8685 for motivation. Does not apply to Windows. + `supports_build_time_venv` + +Boolean; default is `True` + + Whether this runtime supports virtualenvs created at build time. + +See {obj}\`PyRuntimeInfo.supports\_build\_time\_venv\` for docs. + +:::{versionadded} 1.5.0 +::: `zip_main_template` [Label](/concepts/labels); default is `"@rules_python//python/private:zip_main_template"` diff --git a/reference/command-line-reference.mdx b/reference/command-line-reference.mdx deleted file mode 100644 index 73ed4cc..0000000 --- a/reference/command-line-reference.mdx +++ /dev/null @@ -1,13974 +0,0 @@ ---- -title: 'Command-Line Reference' ---- - - - -``` -bazel [] [] - -``` - -or - -``` -bazel [] [] -- [] - -``` - -See the [User's Guide](/docs/build#specifying-build-targets) for the -target patterns syntax. - -## Option Syntax - -Options can be passed to Bazel in different ways. Options that require a value -can be passed with either an equals sign or a space: - -``` ---