diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 30f5ea0ce162..a8de771ae5ca 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,8 +16,9 @@ To get an auto-generated PR description you can put "copilot:summary" or "copilo * [ ] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [ ] I've included a screenshot or gif (if applicable) * [ ] I have tested the web demo (if applicable): - * Full build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html) - * Partial build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way + * Using newly built examples: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html) + * Using examples from latest `main` build: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) + * Using full set of examples from `nightly` build: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [ ] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }}) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000000..51e4ad64d56f --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,55 @@ +name: Nightly + +on: + workflow_dispatch: + schedule: + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule + # 12:15 UTC, every day + - cron: "15 12 * * *" + +jobs: + build-web: + name: "Build Web" + uses: ./.github/workflows/reusable_build_web.yml + with: + CONCURRENCY: nightly + CHANNEL: nightly + secrets: inherit + + upload-web: + name: "Upload Web" + needs: [build-web] + uses: ./.github/workflows/reusable_upload_web.yml + with: + CONCURRENCY: nightly + NIGHTLY: true + secrets: inherit + + build-wheel-linux: + name: "Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: nightly-linux + PLATFORM: linux + WHEEL_ARTIFACT_NAME: linux-wheel + MODE: "pr" + secrets: inherit + + build-examples: + name: "Build Examples" + needs: [build-wheel-linux] + uses: ./.github/workflows/reusable_build_examples.yml + with: + CONCURRENCY: nightly + CHANNEL: nightly + WHEEL_ARTIFACT_NAME: linux-wheel + secrets: inherit + + upload-examples: + name: "Upload Examples" + needs: [build-examples] + uses: ./.github/workflows/reusable_upload_examples.yml + with: + CONCURRENCY: nightly + NIGHTLY: true + secrets: inherit diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index b68683339b48..d5df679648c7 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -111,6 +111,7 @@ jobs: uses: ./.github/workflows/reusable_build_web.yml with: CONCURRENCY: pr-${{ github.event.pull_request.number }} + CHANNEL: main secrets: inherit upload-web: @@ -130,13 +131,14 @@ jobs: uses: ./.github/workflows/reusable_build_examples.yml with: CONCURRENCY: pr-${{ github.event.pull_request.number }} + CHANNEL: main WHEEL_ARTIFACT_NAME: linux-wheel-fast secrets: inherit track-sizes: name: "Track Sizes" if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - needs: [build-examples] + needs: [build-web, build-examples] uses: ./.github/workflows/reusable_track_size.yml with: CONCURRENCY: push-${{ github.ref_name }} diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml index 4915956da4c8..4f623d077c65 100644 --- a/.github/workflows/on_push_main.yml +++ b/.github/workflows/on_push_main.yml @@ -56,7 +56,7 @@ jobs: uses: ./.github/workflows/reusable_deploy_docs.yml with: CONCURRENCY: push-${{ github.ref_name }} - PY_AND_CPP_DOCS_VERSION_NAME: "nightly" + PY_AND_CPP_DOCS_VERSION_NAME: "main" UPDATE_LATEST: false secrets: inherit @@ -65,6 +65,7 @@ jobs: uses: ./.github/workflows/reusable_build_web.yml with: CONCURRENCY: push-${{ github.ref_name }} + CHANNEL: main secrets: inherit upload-web: @@ -81,12 +82,13 @@ jobs: uses: ./.github/workflows/reusable_build_examples.yml with: CONCURRENCY: push-${{ github.ref_name }} + CHANNEL: main WHEEL_ARTIFACT_NAME: linux-wheel secrets: inherit track-sizes: name: "Track Sizes" - needs: [build-examples] + needs: [build-web, build-examples] uses: ./.github/workflows/reusable_track_size.yml with: CONCURRENCY: push-${{ github.ref_name }} diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index 0bf2cce197c6..ef5a9351cefb 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -9,13 +9,15 @@ on: WHEEL_ARTIFACT_NAME: required: true type: string + CHANNEL: # `nightly` or `main` + required: true + type: string concurrency: group: ${{ inputs.CONCURRENCY }}-build-examples cancel-in-progress: true env: - PYTHON_VERSION: "3.8" # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html @@ -33,9 +35,6 @@ jobs: runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.11.0 - steps: - uses: actions/checkout@v4 with: @@ -49,20 +48,26 @@ jobs: workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + - uses: prefix-dev/setup-pixi@v0.4.1 + with: + pixi-version: v0.6.0 + - name: Download Wheel uses: actions/download-artifact@v3 with: name: ${{ inputs.WHEEL_ARTIFACT_NAME }} path: wheel - - uses: prefix-dev/setup-pixi@v0.4.1 - with: - pixi-version: v0.6.0 + - name: Install example dependencies + shell: bash + run: | + pixi run pip install \ + -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt \ + --no-input - name: Install Python dependencies and wheel shell: bash run: | - pixi run pip install -r scripts/ci/requirements-web-demo.txt pixi run pip uninstall rerun-sdk -y pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 pixi run pip install rerun-sdk --no-index --find-links wheel @@ -84,7 +89,9 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples example_data + pixi run build-examples rrd \ + --channel ${{ inputs.CHANNEL }} \ + example_data - name: Upload assets uses: actions/upload-artifact@v3 diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 584b9e57aca6..518a967cef9a 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -10,10 +10,9 @@ on: required: false type: string default: "prerelease" - EXTRA_FLAGS: - required: false + CHANNEL: # `nightly` or `main` + required: true type: string - default: "" concurrency: group: ${{ inputs.CONCURRENCY }}-build-web @@ -54,21 +53,29 @@ jobs: workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + - uses: prefix-dev/setup-pixi@v0.4.1 + with: + pixi-version: v0.6.0 + - name: Build web-viewer (release) shell: bash run: | - cargo run --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }} + if [ ${{ inputs.CHANNEL }} = "nightly" ]; then + export DEFAULT_EXAMPLES_MANIFEST_URL="https://app.rerun.io/version/nightly/examples_manifest.json" + fi + cargo run --locked -p re_build_web_viewer -- --release # We build a single manifest pointing to the `commit` - # All the `pr`, `nightly`, release tag, etc. variants will always just point to the resolved commit + # All the `pr`, `main`, release tag, etc. variants will always just point to the resolved commit - name: Build examples manifest shell: bash run: | full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" sha="$(echo $full_commit | cut -c1-7)" - cargo run --locked -p re_build_examples_manifest -- \ + pixi run build-examples manifest \ --base-url "https://app.rerun.io/commit/$sha" \ + --channel "${{ inputs.CHANNEL }}" \ web_viewer/examples_manifest.json - name: Upload web assets diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml index b8884a5c4df2..8e8f0d633855 100644 --- a/.github/workflows/reusable_publish_web.yml +++ b/.github/workflows/reusable_publish_web.yml @@ -107,7 +107,9 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples web_viewer/examples + pixi run build-examples rrd \ + --channel "main" \ + web_viewer/examples - name: Build examples manifest shell: bash @@ -115,8 +117,9 @@ jobs: full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" sha="$(echo $full_commit | cut -c1-7)" - pixi run build-examples-manifest \ + pixi run build-examples manifest \ --base-url "https://app.rerun.io/commit/$sha" \ + --channel "main" \ web_viewer/examples_manifest.json - name: Upload app.rerun.io (versioned) diff --git a/.github/workflows/reusable_upload_examples.yml b/.github/workflows/reusable_upload_examples.yml index c9b3550917be..cfbd28f8e29c 100644 --- a/.github/workflows/reusable_upload_examples.yml +++ b/.github/workflows/reusable_upload_examples.yml @@ -21,6 +21,10 @@ on: PR_NUMBER: type: string default: "" + NIGHTLY: + required: false + type: boolean + default: false concurrency: group: ${{ inputs.CONCURRENCY }}-upload-examples @@ -61,6 +65,7 @@ jobs: echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: "Upload examples (commit)" + if: ${{ !inputs.NIGHTLY }} uses: google-github-actions/upload-cloud-storage@v1 with: path: "example_data" @@ -93,13 +98,12 @@ jobs: headers: |- cache-control: no-cache, max-age=0 - - name: "Upload examples (nightly)" - # TEMP: Tracking `main` until we can do actual nightly builds. + - name: "Upload examples (main)" if: github.ref == 'refs/heads/main' uses: google-github-actions/upload-cloud-storage@v1 with: path: "example_data" - destination: "rerun-web-viewer/version/nightly/examples" + destination: "rerun-web-viewer/version/main/examples" parent: false headers: |- cache-control: no-cache, max-age=0 @@ -113,3 +117,13 @@ jobs: parent: false headers: |- cache-control: no-cache, max-age=0 + + - name: "Upload examples (nightly)" + if: ${{ inputs.NIGHTLY }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "example_data" + destination: "rerun-web-viewer/version/nightly/examples" + parent: false + headers: |- + cache-control: no-cache, max-age=0 diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index db29e82c37f4..b2a43a745611 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -22,6 +22,10 @@ on: required: false type: string default: "" + NIGHTLY: + required: false + type: boolean + default: false concurrency: group: ${{ inputs.CONCURRENCY }}-upload-web @@ -62,6 +66,7 @@ jobs: echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: "Upload web-viewer (commit)" + if: ${{ !inputs.NIGHTLY }} uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" @@ -94,13 +99,12 @@ jobs: headers: |- cache-control: no-cache, max-age=0 - - name: "Upload web-viewer (nightly)" - # TEMP: Tracking `main` until we can do actual nightly builds. + - name: "Upload web-viewer (main)" if: github.ref == 'refs/heads/main' uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" - destination: "rerun-web-viewer/version/nightly" + destination: "rerun-web-viewer/version/main" parent: false headers: |- cache-control: no-cache, max-age=0 @@ -114,3 +118,13 @@ jobs: parent: false headers: |- cache-control: no-cache, max-age=0 + + - name: "Upload web-viewer (nightly)" + if: ${{ inputs.NIGHTLY }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/version/nightly" + parent: false + headers: |- + cache-control: no-cache, max-age=0 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a707f9cff831..e377c66a1c67 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -179,8 +179,7 @@ Update instructions: | re_build_info | Information about the build. Use together with re_build_tools | | re_build_tools | build.rs helpers for generating build info | | re_types_builder | Generates code for Rerun's SDKs from flatbuffers definitions. | -| re_build_examples | Build rerun example RRD files | -| re_build_examples_manifest | Build the rerun examples manifest JSON file | +| re_build_examples | Build rerun example RRD files and manifest | | re_build_web_viewer | Build the rerun web-viewer Wasm from source | diff --git a/Cargo.lock b/Cargo.lock index b4265b383566..b9a04f1dcee7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,6 +256,37 @@ dependencies = [ "x11rb", ] +[[package]] +name = "argh" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af5ba06967ff7214ce4c7419c7d185be7ecd6cc4965a8f6e1d8ce0398aad219" +dependencies = [ + "argh_derive", + "argh_shared", +] + +[[package]] +name = "argh_derive" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56df0aeedf6b7a2fc67d06db35b09684c3e8da0c95f8f27685cb17e08413d87a" +dependencies = [ + "argh_shared", + "proc-macro2", + "quote", + "syn 2.0.32", +] + +[[package]] +name = "argh_shared" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5693f39141bda5760ecc4111ab08da40565d1771038c4a0250f03457ec707531" +dependencies = [ + "serde", +] + [[package]] name = "argminmax" version = "0.6.1" @@ -4387,19 +4418,11 @@ name = "re_build_examples" version = "0.12.0-alpha.1+dev" dependencies = [ "anyhow", + "argh", "indicatif", "rayon", "re_build_tools", - "serde", - "serde_yaml", -] - -[[package]] -name = "re_build_examples_manifest" -version = "0.12.0-alpha.1+dev" -dependencies = [ - "anyhow", - "re_build_tools", + "re_format", "serde", "serde_json", "serde_yaml", diff --git a/Cargo.toml b/Cargo.toml index b7e2f43c2969..fa4494ddcf53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,6 @@ version = "0.12.0-alpha.1+dev" re_analytics = { path = "crates/re_analytics", version = "=0.12.0-alpha.1", default-features = false } re_arrow_store = { path = "crates/re_arrow_store", version = "=0.12.0-alpha.1", default-features = false } re_build_examples = { path = "crates/re_build_examples", version = "=0.12.0-alpha.1", default-features = false } -re_build_examples_manifest = { path = "crates/re_build_examples_manifest", version = "=0.12.0-alpha.1", default-features = false } re_build_info = { path = "crates/re_build_info", version = "=0.12.0-alpha.1", default-features = false } re_build_tools = { path = "crates/re_build_tools", version = "=0.12.0-alpha.1", default-features = false } re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "=0.12.0-alpha.1", default-features = false } @@ -98,6 +97,7 @@ emath = "0.24.1" ahash = "0.8" anyhow = "1.0" arboard = { version = "3.2", default-features = false } +argh = "0.1.12" array-init = "2.1" arrow2 = "0.17" arrow2_convert = "0.5.0" diff --git a/crates/re_build_examples/Cargo.toml b/crates/re_build_examples/Cargo.toml index 674119cb72d8..71c1a63cf7ff 100644 --- a/crates/re_build_examples/Cargo.toml +++ b/crates/re_build_examples/Cargo.toml @@ -18,10 +18,13 @@ all-features = true [dependencies] re_build_tools.workspace = true +re_format.workspace = true # External anyhow.workspace = true +argh.workspace = true indicatif.workspace = true rayon.workspace = true serde = { workspace = true, features = ["derive"] } serde_yaml.workspace = true +serde_json.workspace = true diff --git a/crates/re_build_examples/src/example.rs b/crates/re_build_examples/src/example.rs new file mode 100644 index 000000000000..b27ad010fa06 --- /dev/null +++ b/crates/re_build_examples/src/example.rs @@ -0,0 +1,166 @@ +//! Example collection and parsing. + +use std::fmt::Display; +use std::path::Path; +use std::path::PathBuf; +use std::str::FromStr; + +pub struct Example { + pub name: String, + pub title: String, + pub description: String, + pub tags: Vec, + pub thumbnail_url: String, + pub thumbnail_dimensions: [u64; 2], + pub script_path: PathBuf, + pub script_args: Vec, +} + +#[derive(Default, Clone, Copy, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Channel { + #[default] + Main, + Nightly, +} + +impl Channel { + pub fn examples(self) -> anyhow::Result> { + let mut examples = vec![]; + let dir = Path::new("examples/python"); + if !dir.exists() { + anyhow::bail!("Failed to find {}", dir.display()) + } + if !dir.is_dir() { + anyhow::bail!("{} is not a directory", dir.display()) + } + + let folders: std::collections::BTreeMap = + std::fs::read_dir(dir)? + .filter_map(Result::ok) + .map(|folder| { + let name = folder.file_name().to_string_lossy().to_string(); + (name, folder) + }) + .collect(); + + for (name, folder) in folders { + let metadata = folder.metadata()?; + let readme = folder.path().join("README.md"); + if metadata.is_dir() && readme.exists() { + let readme = parse_frontmatter(readme)?; + let Some(readme) = readme else { + eprintln!("{name:?}: skipped - MISSING FRONTMATTER"); + continue; + }; + + let Some(channel) = readme.channel else { + eprintln!("{name:?}: skipped"); + continue; + }; + + if channel != self { + eprintln!("{name:?}: skipped"); + continue; + } + + eprintln!("{name:?}: added"); + examples.push(Example { + name, + title: readme.title, + description: readme.description, + tags: readme.tags, + thumbnail_url: readme.thumbnail, + thumbnail_dimensions: readme.thumbnail_dimensions, + script_path: folder.path().join("main.py"), + script_args: readme.build_args, + }); + } + } + + if examples.is_empty() { + anyhow::bail!("No examples found in {}", dir.display()) + } + + examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(examples) + } +} + +impl Display for Channel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + Channel::Main => "main", + Channel::Nightly => "nightly", + }; + f.write_str(s) + } +} + +impl FromStr for Channel { + type Err = InvalidChannelName; + + fn from_str(s: &str) -> Result { + match s { + "main" => Ok(Self::Main), + "nightly" => Ok(Self::Nightly), + _ => Err(InvalidChannelName), + } + } +} + +#[derive(Debug)] +pub struct InvalidChannelName; + +impl std::error::Error for InvalidChannelName {} + +impl Display for InvalidChannelName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("invalid channel name") + } +} + +#[derive(serde::Deserialize)] +struct Frontmatter { + #[serde(default)] + title: String, + + #[serde(default)] + tags: Vec, + + #[serde(default)] + description: String, + + #[serde(default)] + thumbnail: String, + + #[serde(default)] + thumbnail_dimensions: [u64; 2], + + #[serde(default)] + channel: Option, + + #[serde(default)] + build_args: Vec, +} + +fn parse_frontmatter>(path: P) -> anyhow::Result> { + let path = path.as_ref(); + let content = std::fs::read_to_string(path)?; + let content = content.replace('\r', ""); // Windows, god damn you + re_build_tools::rerun_if_changed(path); + let Some(content) = content.strip_prefix("---\n") else { + return Ok(None); + }; + let Some(end) = content.find("---") else { + anyhow::bail!("{:?} has invalid frontmatter: missing --- terminator", path); + }; + Ok(Some(serde_yaml::from_str(&content[..end]).map_err( + |err| { + anyhow::anyhow!( + "failed to read {:?}: {err}", + path.parent().unwrap().file_name().unwrap() + ) + }, + )?)) +} diff --git a/crates/re_build_examples/src/main.rs b/crates/re_build_examples/src/main.rs index 945320115902..0931dc73e203 100644 --- a/crates/re_build_examples/src/main.rs +++ b/crates/re_build_examples/src/main.rs @@ -1,239 +1,42 @@ -//! This build script collects all examples which should be part of our example page -//! and runs them to produce `.rrd` files. +//! This build script collects all examples which should be part of our example page, +//! and either runs them to produce `.rrd` files, or builds a manifest file which +//! serves as an index for the files. +//! +//! It identifies runnable examples by checking if they have `channel` set in +//! their `README.md` frontmatter. The available values are: +//! - `main` for simple/fast examples built on each PR and the `main` branch +//! - `nightly` for heavier examples built once per day //! -//! It identifies runnable examples by checking if they have `demo: true` set in -//! their `README.md` frontmatter. //! An example may also specify args to be run with via the frontmatter //! `build_args` string array. -use std::fs::create_dir_all; -use std::fs::read_dir; -use std::fs::read_to_string; -use std::io::stdout; -use std::io::IsTerminal; -use std::path::Path; -use std::path::PathBuf; -use std::process::exit; -use std::process::Command; -use std::process::Output; -use std::time::Duration; - -use indicatif::MultiProgress; -use indicatif::ProgressBar; -use rayon::prelude::IntoParallelIterator; -use rayon::prelude::ParallelIterator; - -const USAGE: &str = "\ -Usage: [options] [output_dir] +mod example; +mod manifest; +mod rrd; -Options: - -h, --help Print help -"; +use argh::FromArgs; +use example::{Channel, Example}; fn main() -> anyhow::Result<()> { re_build_tools::set_output_cargo_build_instructions(false); - let args = Args::from_env(); - - create_dir_all(&args.output_dir)?; - - let examples = examples()?; - let progress = MultiProgress::new(); - let results: Vec> = examples - .into_par_iter() - .map(|example| example.run(&progress, &args.output_dir)) - .collect(); - - let mut failed = false; - for result in results { - if let Err(err) = result { - eprintln!("{err}"); - failed = true; - } + let args: Args = argh::from_env(); + match args.cmd { + Cmd::Rrd(cmd) => cmd.run(), + Cmd::Manifest(cmd) => cmd.run(), } - if failed { - anyhow::bail!("Failed to run some examples"); - } - - Ok(()) } +/// Build examples and their manifest. +#[derive(FromArgs)] struct Args { - output_dir: PathBuf, -} - -impl Args { - fn from_env() -> Self { - let mut output_dir = None; - - for arg in std::env::args().skip(1) { - match arg.as_str() { - "--help" | "-h" => { - println!("{USAGE}"); - exit(1); - } - _ if arg.starts_with('-') => { - eprintln!("Unknown argument: {arg:?}"); - println!("\n{USAGE}"); - exit(1); - } - _ if output_dir.is_some() => { - eprintln!("Too many positional arguments"); - println!("\n{USAGE}"); - exit(1); - } - _ => output_dir = Some(PathBuf::from(arg)), - } - } - - let Some(output_dir) = output_dir else { - eprintln!("Missing argument \"output_dir\""); - exit(1); - }; - - Args { output_dir } - } -} - -#[derive(serde::Deserialize)] -struct Frontmatter { - #[serde(default)] - demo: bool, - #[serde(default)] - build_args: Vec, -} - -struct Example { - name: String, - script_path: PathBuf, - script_args: Vec, -} - -impl Example { - fn run(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result<()> { - let mut cmd = Command::new("python3"); - cmd.arg(self.script_path); - cmd.arg("--save") - .arg(output_dir.join(&self.name).with_extension("rrd")); - cmd.args(self.script_args); - - let final_args = cmd - .get_args() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - // Configure flushing so that: - // * the resulting file size is deterministic - // * the file is chunked into small batches for better streaming - cmd.env("RERUN_FLUSH_TICK_SECS", 1_000_000_000.to_string()); - cmd.env("RERUN_FLUSH_NUM_BYTES", (128 * 1024).to_string()); - - let output = wait_for_output(cmd, &self.name, progress)?; - - if !output.status.success() { - anyhow::bail!( - "Failed to run `python3 {}`: \ - \nstdout: \ - \n{} \ - \nstderr: \ - \n{}", - final_args.join(" "), - String::from_utf8(output.stdout)?, - String::from_utf8(output.stderr)?, - ); - } - - Ok(()) - } -} - -fn wait_for_output( - mut cmd: Command, - name: &str, - progress: &MultiProgress, -) -> anyhow::Result { - let progress = progress.add(ProgressBar::new_spinner().with_message(name.to_owned())); - progress.enable_steady_tick(Duration::from_millis(100)); - - let output = cmd.output()?; - - let elapsed = progress.elapsed().as_secs_f64(); - let tick = if output.status.success() { - "✔" - } else { - "✘" - }; - let message = format!("{tick} {name} ({elapsed:.3}s)"); - - if stdout().is_terminal() { - progress.set_message(message); - progress.finish(); - } else { - println!("{message}"); - } - - Ok(output) -} - -fn examples() -> anyhow::Result> { - let mut examples = vec![]; - let dir = Path::new("examples/python"); - if !dir.exists() { - anyhow::bail!("Failed to find {}", dir.display()) - } - if !dir.is_dir() { - anyhow::bail!("{} is not a directory", dir.display()) - } - - for folder in read_dir(dir)? { - let folder = folder?; - let metadata = folder.metadata()?; - let name = folder.file_name().to_string_lossy().to_string(); - let readme = folder.path().join("README.md"); - if metadata.is_dir() && readme.exists() { - let readme = parse_frontmatter(readme)?; - if let Some(readme) = readme { - if readme.demo { - eprintln!("Adding example {name:?}"); - examples.push(Example { - name, - script_path: folder.path().join("main.py"), - script_args: readme.build_args, - }); - } else { - eprintln!("Skipping example {name:?} because 'demo' is set to 'false'"); - } - } else { - eprintln!("Skipping example {name:?} because it has no frontmatter"); - } - } - } - - if examples.is_empty() { - anyhow::bail!("No examples found in {}", dir.display()) - } - - examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); - Ok(examples) + #[argh(subcommand)] + cmd: Cmd, } -fn parse_frontmatter>(path: P) -> anyhow::Result> { - let path = path.as_ref(); - let content = read_to_string(path)?; - let content = content.replace('\r', ""); // Windows, god damn you - re_build_tools::rerun_if_changed(path); - let Some(content) = content.strip_prefix("---\n") else { - return Ok(None); - }; - let Some(end) = content.find("---") else { - anyhow::bail!("{:?} has invalid frontmatter", path); - }; - Ok(Some(serde_yaml::from_str(&content[..end]).map_err( - |e| { - anyhow::anyhow!( - "failed to read {:?}: {e}", - path.parent().unwrap().file_name().unwrap() - ) - }, - )?)) +#[derive(FromArgs)] +#[argh(subcommand)] +enum Cmd { + Rrd(rrd::Rrd), + Manifest(manifest::Manifest), } diff --git a/crates/re_build_examples/src/manifest.rs b/crates/re_build_examples/src/manifest.rs new file mode 100644 index 000000000000..b1b5f11cf7d6 --- /dev/null +++ b/crates/re_build_examples/src/manifest.rs @@ -0,0 +1,149 @@ +use re_build_tools::Environment; + +use crate::{Channel, Example}; +use std::path::PathBuf; + +/// Collect examples in the repository and produce a manifest file. +/// +/// The manifest file contains example metadata, such as their names +/// and links to `.rrd` files. +#[derive(argh::FromArgs)] +#[argh(subcommand, name = "manifest")] +pub struct Manifest { + #[argh(positional, description = "output path for the manifest file")] + output_path: PathBuf, + + #[argh( + option, + description = "where examples are uploaded, e.g. `https://app.rerun.io/version/main`" + )] + base_url: Option, + + #[argh(option, description = "include only examples in this channel")] + channel: Channel, +} + +impl Manifest { + pub fn run(self) -> anyhow::Result<()> { + let build_env = Environment::detect(); + + let base_url = if matches!(self.channel, Channel::Nightly) { + "https://app.rerun.io/version/nightly".to_owned() + } else { + match &self.base_url { + Some(base_url) => base_url.clone(), + None => get_base_url(build_env)?, + } + }; + + let manifest = self + .channel + .examples()? + .into_iter() + .map(|example| ManifestEntry::new(example, &base_url)) + .collect::>(); + + if manifest.is_empty() { + anyhow::bail!("No examples found!"); + } + + std::fs::write(self.output_path, serde_json::to_string_pretty(&manifest)?)?; + + Ok(()) + } +} + +#[derive(serde::Serialize)] +struct ManifestEntry { + name: String, + title: String, + description: String, + tags: Vec, + rrd_url: String, + thumbnail: Thumbnail, +} + +impl ManifestEntry { + fn new(example: Example, base_url: &str) -> Self { + let name = example.name; + Self { + title: example.title, + description: example.description, + tags: example.tags, + rrd_url: format!("{base_url}/examples/{name}.rrd"), + thumbnail: Thumbnail { + url: example.thumbnail_url, + width: example.thumbnail_dimensions[0], + height: example.thumbnail_dimensions[1], + }, + name, + } + } +} + +#[derive(serde::Serialize)] +struct Thumbnail { + url: String, + width: u64, + height: u64, +} + +fn get_base_url(build_env: Environment) -> anyhow::Result { + // In the CondaBuild environment we can't trust the git_branch name -- if it exists + // at all it's going to be the feedstock branch-name, not our Rerun branch. However + // conda should ONLY be building released versions, so we want to version the manifest. + let versioned_manifest = matches!(build_env, Environment::CondaBuild) || { + let branch = re_build_tools::git_branch()?; + if branch == "main" || !re_build_tools::is_on_ci() { + // on `main` and local builds, use `version/main` + // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` + // on every commit to the `main` branch + return Ok("https://app.rerun.io/version/main".into()); + } + parse_release_version(&branch).is_some() + }; + + if versioned_manifest { + let metadata = re_build_tools::cargo_metadata()?; + let workspace_root = metadata + .root_package() + .ok_or_else(|| anyhow::anyhow!("failed to find workspace root"))?; + + // on `release-x.y.z` builds, use `version/{crate_version}` + // this will point to data uploaded by `.github/workflows/reusable_build_and_publish_web.yml` + return Ok(format!( + "https://app.rerun.io/version/{}", + workspace_root.version + )); + } + + // any other branch that is not `main`, use `commit/{sha}` + // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` + let sha = re_build_tools::git_commit_short_hash()?; + Ok(format!("https://app.rerun.io/commit/{sha}")) +} + +fn parse_release_version(branch: &str) -> Option<&str> { + // release-\d+.\d+.\d+(-alpha.\d+)? + + let version = branch.strip_prefix("release-")?; + + let (major, rest) = version.split_once('.')?; + major.parse::().ok()?; + let (minor, rest) = rest.split_once('.')?; + minor.parse::().ok()?; + let (patch, meta) = rest + .split_once('-') + .map_or((rest, None), |(p, m)| (p, Some(m))); + patch.parse::().ok()?; + + if let Some(meta) = meta { + let (kind, n) = meta.split_once('.')?; + if kind != "alpha" && kind != "rc" { + return None; + } + n.parse::().ok()?; + } + + Some(version) +} diff --git a/crates/re_build_examples/src/rrd.rs b/crates/re_build_examples/src/rrd.rs new file mode 100644 index 000000000000..25a87dda5e7b --- /dev/null +++ b/crates/re_build_examples/src/rrd.rs @@ -0,0 +1,136 @@ +use crate::{Channel, Example}; +use indicatif::MultiProgress; +use indicatif::ProgressBar; +use rayon::prelude::IntoParallelIterator; +use rayon::prelude::ParallelIterator; +use std::fs::create_dir_all; +use std::io::stdout; +use std::io::IsTerminal; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; +use std::process::Output; +use std::time::Duration; + +/// Collect examples in the repository and run them to produce `.rrd` files. +#[derive(argh::FromArgs)] +#[argh(subcommand, name = "rrd")] +pub struct Rrd { + #[argh(positional, description = "directory to output `rrd` files into")] + output_dir: PathBuf, + + #[argh(option, description = "include only examples in this channel")] + channel: Channel, +} + +impl Rrd { + pub fn run(self) -> anyhow::Result<()> { + create_dir_all(&self.output_dir)?; + + let examples = self.channel.examples()?; + let progress = MultiProgress::new(); + let results: Vec> = examples + .into_par_iter() + .map(|example| example.build(&progress, &self.output_dir)) + .collect(); + + let mut failed = false; + for result in results { + match result { + Ok(rrd_path) => { + if let Ok(metadata) = std::fs::metadata(&rrd_path) { + println!( + "Output: {} ({})", + rrd_path.display(), + re_format::format_bytes(metadata.len() as _) + ); + } else { + eprintln!("Missing rrd at {}", rrd_path.display()); + failed = true; + } + } + Err(err) => { + eprintln!("{err}"); + failed = true; + } + } + } + if failed { + anyhow::bail!("Failed to run some examples"); + } + + Ok(()) + } +} + +trait Build { + /// Returns the path to the resulting `.rrd` file. + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result; +} + +impl Build for Example { + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result { + let rrd_path = output_dir.join(&self.name).with_extension("rrd"); + + let mut cmd = Command::new("python3"); + cmd.arg(self.script_path); + cmd.arg("--save").arg(&rrd_path); + cmd.args(self.script_args); + + let final_args = cmd + .get_args() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + // Configure flushing so that: + // * the resulting file size is deterministic + // * the file is chunked into small batches for better streaming + cmd.env("RERUN_FLUSH_TICK_SECS", 1_000_000_000.to_string()); + cmd.env("RERUN_FLUSH_NUM_BYTES", (128 * 1024).to_string()); + + let output = wait_for_output(cmd, &self.name, progress)?; + + if output.status.success() { + Ok(rrd_path) + } else { + anyhow::bail!( + "Failed to run `python3 {}`: \ + \nstdout: \ + \n{} \ + \nstderr: \ + \n{}", + final_args.join(" "), + String::from_utf8(output.stdout)?, + String::from_utf8(output.stderr)?, + ); + } + } +} + +fn wait_for_output( + mut cmd: Command, + name: &str, + progress: &MultiProgress, +) -> anyhow::Result { + let progress = progress.add(ProgressBar::new_spinner().with_message(name.to_owned())); + progress.enable_steady_tick(Duration::from_millis(100)); + + let output = cmd.output()?; + + let elapsed = progress.elapsed().as_secs_f64(); + let tick = if output.status.success() { + "✔" + } else { + "✘" + }; + let message = format!("{tick} {name} ({elapsed:.3}s)"); + + if stdout().is_terminal() { + progress.set_message(message); + progress.finish(); + } else { + println!("{message}"); + } + + Ok(output) +} diff --git a/crates/re_build_examples_manifest/Cargo.toml b/crates/re_build_examples_manifest/Cargo.toml deleted file mode 100644 index ddc43a09e4a1..000000000000 --- a/crates/re_build_examples_manifest/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "re_build_examples_manifest" -authors.workspace = true -description = "Build the rerun examples manifest JSON file" -edition.workspace = true -homepage.workspace = true -include.workspace = true -license.workspace = true -publish = false -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[package.metadata.docs.rs] -all-features = true - - -[dependencies] -re_build_tools.workspace = true - -# External -anyhow.workspace = true -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true -serde_yaml.workspace = true diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs deleted file mode 100644 index 310e756971e0..000000000000 --- a/crates/re_build_examples_manifest/src/main.rs +++ /dev/null @@ -1,282 +0,0 @@ -//! This build script generates the `examples_manifest.json` file. -//! It looks at all examples in the workspace (under `examples/python`), -//! and only includes those with `demo` set to `true` in their `README.md` -//! frontmatter. -//! -//! The URLs embedded in the `example_manifest.json` file point to a specific version. -//! This version is resolved according to the current environment: -//! -//! If the `CI` env var is set + the branch name is not `main`, then: -//! - On any `release-x.y.z` branch, the version is `version/x.y.z` -//! - On any other branch, the version is `commit/$COMMIT_SHORT_HASH` -//! -//! Otherwise, the version is `version/nightly`. This means local builds, -//! and builds on `main` point to `version/nightly`. - -use std::path::Path; -use std::path::PathBuf; - -use re_build_tools::Environment; - -const USAGE: &str = "\ -Usage: [options] [output_path] - -Options: - -h, --help Print help - --base-url Where all examples are uploaded, e.g. `https://demo.rerun.io/version/nightly`. -"; - -fn main() -> anyhow::Result<()> { - re_build_tools::set_output_cargo_build_instructions(false); - - let args = Args::from_env(); - - let manifest = build_examples_manifest(Environment::detect(), &args)?; - std::fs::write(args.output_path, manifest)?; - - Ok(()) -} - -struct Args { - output_path: PathBuf, - base_url: Option, -} - -impl Args { - fn from_env() -> Self { - let mut output_path = None; - let mut base_url = None; - - let mut args = std::env::args().skip(1); - while let Some(arg) = args.next() { - match arg.as_str() { - "--help" | "-h" => { - println!("{USAGE}"); - std::process::exit(1); - } - "--base-url" => { - let Some(url) = args.next() else { - eprintln!("Expected value after \"--base-url\""); - println!("\n{USAGE}"); - std::process::exit(1); - }; - base_url = Some(url); - } - _ if arg.starts_with('-') => { - eprintln!("Unknown argument: {arg:?}"); - println!("\n{USAGE}"); - std::process::exit(1); - } - _ if output_path.is_some() => { - eprintln!("Too many positional arguments"); - println!("\n{USAGE}"); - std::process::exit(1); - } - _ => output_path = Some(PathBuf::from(arg)), - } - } - - let Some(output_path) = output_path else { - eprintln!("Missing argument \"output_path\""); - std::process::exit(1); - }; - - Args { - output_path, - base_url, - } - } -} - -fn build_examples_manifest(build_env: Environment, args: &Args) -> anyhow::Result { - let base_url = match &args.base_url { - Some(base_url) => base_url.clone(), - None => get_base_url(build_env)?, - }; - - let mut manifest = vec![]; - for example in examples()? { - manifest.push(ManifestEntry::new(example, &base_url)); - } - - if manifest.is_empty() { - anyhow::bail!("No examples found!"); - } - - Ok(serde_json::to_string_pretty(&manifest)?) -} - -#[derive(serde::Deserialize)] -struct Frontmatter { - #[serde(default)] - title: String, - #[serde(default)] - tags: Vec, - #[serde(default)] - description: String, - #[serde(default)] - thumbnail: String, - #[serde(default)] - thumbnail_dimensions: [u64; 2], - #[serde(default)] - demo: bool, -} - -#[derive(serde::Serialize)] -struct ManifestEntry { - name: String, - title: String, - description: String, - tags: Vec, - rrd_url: String, - thumbnail: Thumbnail, -} - -impl ManifestEntry { - fn new(example: Example, base_url: &str) -> Self { - let Example { name, readme } = example; - Self { - title: readme.title, - description: readme.description, - tags: readme.tags, - rrd_url: format!("{base_url}/examples/{name}.rrd"), - thumbnail: Thumbnail { - url: readme.thumbnail, - width: readme.thumbnail_dimensions[0], - height: readme.thumbnail_dimensions[1], - }, - name, - } - } -} - -#[derive(serde::Serialize)] -struct Thumbnail { - url: String, - width: u64, - height: u64, -} - -struct Example { - name: String, - readme: Frontmatter, -} - -fn examples() -> anyhow::Result> { - let mut examples = vec![]; - let dir = Path::new("examples/python"); - if !dir.exists() { - anyhow::bail!("Failed to find {}", dir.display()) - } - if !dir.is_dir() { - anyhow::bail!("{} is not a directory", dir.display()) - } - - for folder in std::fs::read_dir(dir)? { - let folder = folder?; - let metadata = folder.metadata()?; - let name = folder.file_name().to_string_lossy().to_string(); - let readme = folder.path().join("README.md"); - if metadata.is_dir() && readme.exists() { - let readme = parse_frontmatter(readme)?; - if let Some(readme) = readme { - if readme.demo { - eprintln!("Adding example {name:?}"); - examples.push(Example { name, readme }); - } else { - eprintln!("Skipping example {name:?} because 'demo' is set to 'false'"); - } - } else { - eprintln!("Skipping example {name:?} because it has no frontmatter"); - } - } - } - - if examples.is_empty() { - anyhow::bail!("No examples found in {}", dir.display()) - } - - examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); - Ok(examples) -} - -fn parse_frontmatter>(path: P) -> anyhow::Result> { - let path = path.as_ref(); - let content = std::fs::read_to_string(path)?; - let content = content.replace('\r', ""); // Windows, god damn you - re_build_tools::rerun_if_changed(path); - let Some(content) = content.strip_prefix("---\n") else { - return Ok(None); - }; - let Some(end) = content.find("---") else { - anyhow::bail!("{:?} has invalid frontmatter", path); - }; - Ok(Some(serde_yaml::from_str(&content[..end]).map_err( - |e| { - anyhow::anyhow!( - "failed to read {:?}: {e}", - path.parent().unwrap().file_name().unwrap() - ) - }, - )?)) -} - -fn get_base_url(build_env: Environment) -> anyhow::Result { - // In the CondaBuild environment we can't trust the git_branch name -- if it exists - // at all it's going to be the feedstock branch-name, not our Rerun branch. However - // conda should ONLY be building released versions, so we want to version the manifest. - let versioned_manifest = matches!(build_env, Environment::CondaBuild) || { - let branch = re_build_tools::git_branch()?; - if branch == "main" || !re_build_tools::is_on_ci() { - // on `main` and local builds, use `version/nightly` - // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` - // on every commit to the `main` branch - return Ok("https://app.rerun.io/version/nightly".into()); - } - parse_release_version(&branch).is_some() - }; - - if versioned_manifest { - let metadata = re_build_tools::cargo_metadata()?; - let workspace_root = metadata - .root_package() - .ok_or_else(|| anyhow::anyhow!("failed to find workspace root"))?; - - // on `release-x.y.z` builds, use `version/{crate_version}` - // this will point to data uploaded by `.github/workflows/reusable_build_and_publish_web.yml` - return Ok(format!( - "https://app.rerun.io/version/{}", - workspace_root.version - )); - } - - // any other branch that is not `main`, use `commit/{sha}` - // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` - let sha = re_build_tools::git_commit_short_hash()?; - Ok(format!("https://app.rerun.io/commit/{sha}")) -} - -fn parse_release_version(branch: &str) -> Option<&str> { - // release-\d+.\d+.\d+(-alpha.\d+)? - - let version = branch.strip_prefix("release-")?; - - let (major, rest) = version.split_once('.')?; - major.parse::().ok()?; - let (minor, rest) = rest.split_once('.')?; - minor.parse::().ok()?; - let (patch, meta) = rest - .split_once('-') - .map_or((rest, None), |(p, m)| (p, Some(m))); - patch.parse::().ok()?; - - if let Some(meta) = meta { - let (kind, n) = meta.split_once('.')?; - if kind != "alpha" && kind != "rc" { - return None; - } - n.parse::().ok()?; - } - - Some(version) -} diff --git a/crates/re_viewer/src/ui/welcome_screen/example_page.rs b/crates/re_viewer/src/ui/welcome_screen/example_page.rs index 9fbfa1617362..f145d5ab3832 100644 --- a/crates/re_viewer/src/ui/welcome_screen/example_page.rs +++ b/crates/re_viewer/src/ui/welcome_screen/example_page.rs @@ -125,12 +125,17 @@ pub(super) struct ExamplePage { } fn default_manifest_url() -> String { + // Sometimes we want the default to point somewhere else, such as when doing nightly builds. + if let Some(url) = option_env!("DEFAULT_EXAMPLES_MANIFEST_URL") { + return url.into(); + } + let build_info = re_build_info::build_info!(); - // Always point to `version/nightly` for rerun devs, + // Always point to `version/main` for rerun devs, // because the current commit's manifest is unlikely to be uploaded to GCS. if build_info.is_in_rerun_workspace { - return "https://app.rerun.io/version/nightly/examples_manifest.json".into(); + return "https://app.rerun.io/version/main/examples_manifest.json".into(); } // Otherwise point to the current commit diff --git a/examples/README.md b/examples/README.md index ea04a9f1b13f..46d8dc60e6fa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,3 +47,8 @@ The `manifest.yml` file describes the structure of the examples contained in thi You can base your example off of `python/template` or `rust/template`. Once it's ready to be displayed in the docs, add it to the [manifest](./manifest.yml). + +If you want to run the example on CI and include it in the in-viewer example page, +add a `channel` entry to its README frontmatter. The available channels right now are: +- `main` for simple/fast examples built on each PR and the `main` branch +- `nightly` for heavier examples built once per day diff --git a/examples/cpp/dna/README.md b/examples/cpp/dna/README.md index a8a4178932c4..ba50cdc03d01 100644 --- a/examples/cpp/dna/README.md +++ b/examples/cpp/dna/README.md @@ -7,7 +7,7 @@ tags: [3d, api-example] description: "Simple example of logging point and line primitives to draw a 3D helix." thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/examples/python/arkit_scenes/README.md b/examples/python/arkit_scenes/README.md index e2da39fe359e..f45eb2b7e758 100644 --- a/examples/python/arkit_scenes/README.md +++ b/examples/python/arkit_scenes/README.md @@ -5,7 +5,7 @@ tags: [2D, 3D, depth, mesh, object-detection, pinhole-camera] description: "Visualize the ARKitScenes dataset, which contains color+depth images, the reconstructed mesh and labeled bounding boxes." thumbnail: https://static.rerun.io/arkit_scenes/fb9ec9e8d965369d39d51b17fc7fc5bae6be10cc/480w.png thumbnail_dimensions: [480, 243] -demo: true +channel: main --- diff --git a/examples/python/arkit_scenes/requirements.txt b/examples/python/arkit_scenes/requirements.txt index 723970a7529d..76da574205e0 100644 --- a/examples/python/arkit_scenes/requirements.txt +++ b/examples/python/arkit_scenes/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib>=3.8.2 numpy opencv-python pandas diff --git a/examples/python/depth_guided_stable_diffusion/README.md b/examples/python/depth_guided_stable_diffusion/README.md index 9fc78a37491d..94de9d5f6d83 100644 --- a/examples/python/depth_guided_stable_diffusion/README.md +++ b/examples/python/depth_guided_stable_diffusion/README.md @@ -4,6 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/depth_guid tags: [2D, depth, huggingface, stable-diffusion, tensor, text] thumbnail: https://static.rerun.io/depth_guided_stable_diffusion/a85516aba09f72649517891d767e15383ce7f4ea/480w.png thumbnail_dimensions: [480, 253] +channel: nightly --- diff --git a/examples/python/depth_guided_stable_diffusion/requirements.txt b/examples/python/depth_guided_stable_diffusion/requirements.txt index 184d26dd6ab6..642f1c5b000e 100644 --- a/examples/python/depth_guided_stable_diffusion/requirements.txt +++ b/examples/python/depth_guided_stable_diffusion/requirements.txt @@ -1,5 +1,5 @@ accelerate -diffusers>=0.12.1 +diffusers>=0.12.1,<=0.21 ftfy numpy packaging diff --git a/examples/python/detect_and_track_objects/README.md b/examples/python/detect_and_track_objects/README.md index 71a8effa0e0a..61ce69ee6f1b 100644 --- a/examples/python/detect_and_track_objects/README.md +++ b/examples/python/detect_and_track_objects/README.md @@ -5,7 +5,7 @@ tags: [2D, huggingface, object-detection, object-tracking, opencv] description: "Visualize object detection and segmentation using the Huggingface `transformers` library." thumbnail: https://static.rerun.io/detect_and_track_objects/59f5b97a8724f9037353409ab3d0b7cb47d1544b/480w.png thumbnail_dimensions: [480, 279] -demo: true +channel: nightly --- diff --git a/examples/python/dicom_mri/README.md b/examples/python/dicom_mri/README.md index 367fbef7f05a..abdcb0bb9b96 100644 --- a/examples/python/dicom_mri/README.md +++ b/examples/python/dicom_mri/README.md @@ -5,7 +5,7 @@ tags: [tensor, mri, dicom] description: "Example using a DICOM MRI scan. This demonstrates the flexible tensor slicing capabilities of the Rerun viewer." thumbnail: https://static.rerun.io/dicom_mri/e39f34a1b1ddd101545007f43a61783e1d2e5f8e/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/examples/python/dna/README.md b/examples/python/dna/README.md index 05894d5e2295..5bea9a1a365e 100644 --- a/examples/python/dna/README.md +++ b/examples/python/dna/README.md @@ -7,7 +7,7 @@ tags: [3d, api-example] description: "Simple example of logging point and line primitives to draw a 3D helix." thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/examples/python/human_pose_tracking/README.md b/examples/python/human_pose_tracking/README.md index 4241e3b5defc..10d1eb168ac9 100644 --- a/examples/python/human_pose_tracking/README.md +++ b/examples/python/human_pose_tracking/README.md @@ -5,7 +5,7 @@ tags: [mediapipe, keypoint-detection, 2D, 3D] description: "Use the MediaPipe Pose solution to detect and track a human pose in video." thumbnail: https://static.rerun.io/human_pose_tracking/37d47fe7e3476513f9f58c38da515e2cd4a093f9/480w.png thumbnail_dimensions: [480, 272] -demo: true +channel: main --- diff --git a/examples/python/human_pose_tracking/requirements.txt b/examples/python/human_pose_tracking/requirements.txt index 574bdb7345ef..b83d297cf823 100644 --- a/examples/python/human_pose_tracking/requirements.txt +++ b/examples/python/human_pose_tracking/requirements.txt @@ -1,4 +1,4 @@ -mediapipe>=0.10.1 +mediapipe>=0.10.9 numpy opencv-python<4.6 # Avoid opencv-4.6 since it rotates images incorrectly (https://github.com/opencv/opencv/issues/22088) requests>=2.31,<3 diff --git a/examples/python/lidar/requirements.txt b/examples/python/lidar/requirements.txt index 78d507945c75..2a4536107e25 100644 --- a/examples/python/lidar/requirements.txt +++ b/examples/python/lidar/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib>=3.8.2 numpy nuscenes-devkit requests diff --git a/examples/python/nuscenes/README.md b/examples/python/nuscenes/README.md index ad1a7b6d4e5c..72b8a4b8ba98 100644 --- a/examples/python/nuscenes/README.md +++ b/examples/python/nuscenes/README.md @@ -5,6 +5,7 @@ tags: [lidar, 3D, 2D, object-detection, pinhole-camera] description: "Visualize the nuScenes dataset including lidar, radar, images, and bounding boxes." thumbnail: https://static.rerun.io/nuscenes/64a50a9d67cbb69ae872551989ee807b195f6b5d/480w.png thumbnail_dimensions: [480, 282] +channel: nightly --- diff --git a/examples/python/nuscenes/requirements.txt b/examples/python/nuscenes/requirements.txt index 78d507945c75..2a4536107e25 100644 --- a/examples/python/nuscenes/requirements.txt +++ b/examples/python/nuscenes/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib>=3.8.2 numpy nuscenes-devkit requests diff --git a/examples/python/objectron/README.md b/examples/python/objectron/README.md index 77173cb5346c..0464566890d0 100644 --- a/examples/python/objectron/README.md +++ b/examples/python/objectron/README.md @@ -5,6 +5,7 @@ rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/objectron/src/ tags: [2D, 3D, object-detection, pinhole-camera] thumbnail: https://static.rerun.io/objectron/8ea3a37e6b4af2e06f8e2ea5e70c1951af67fea8/480w.png thumbnail_dimensions: [480, 268] +channel: nightly --- diff --git a/examples/python/open_photogrammetry_format/README.md b/examples/python/open_photogrammetry_format/README.md index 87eaadb7b3d3..f9d9c16a6038 100644 --- a/examples/python/open_photogrammetry_format/README.md +++ b/examples/python/open_photogrammetry_format/README.md @@ -4,6 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/open_photo tags: [2d, 3d, camera, photogrammetry] thumbnail: https://static.rerun.io/open_photogrammetry_format/603d5605f9670889bc8bce3365f16b831fce1eb1/480w.png thumbnail_dimensions: [480, 310] +channel: nightly --- diff --git a/examples/python/plots/README.md b/examples/python/plots/README.md index c2582f149f70..d8855202bd35 100644 --- a/examples/python/plots/README.md +++ b/examples/python/plots/README.md @@ -5,7 +5,7 @@ tags: [2d, plots, api-example] description: "Demonstration of various plots and charts supported by Rerun." thumbnail: https://static.rerun.io/plots/c5b91cf0bf2eaf91c71d6cdcd4fe312d4aeac572/480w.png thumbnail_dimensions: [480, 271] -demo: true +channel: main --- diff --git a/examples/python/raw_mesh/README.md b/examples/python/raw_mesh/README.md index f2b8bc16f453..0d905bdab2ec 100644 --- a/examples/python/raw_mesh/README.md +++ b/examples/python/raw_mesh/README.md @@ -5,6 +5,7 @@ rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/raw_mesh/src/m tags: [mesh] thumbnail: https://static.rerun.io/raw_mesh/64bec98280b07794f7c9617f30ba2c20278601c3/480w.png thumbnail_dimensions: [480, 271] +channel: nightly --- diff --git a/examples/python/rgbd/README.md b/examples/python/rgbd/README.md index d2e2d219376b..1bca9fe0cc38 100644 --- a/examples/python/rgbd/README.md +++ b/examples/python/rgbd/README.md @@ -4,6 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/rgbd/main. tags: [2D, 3D, depth, nyud, pinhole-camera] thumbnail: https://static.rerun.io/rgbd/4109d29ed52fa0a8f980fcdd0e9673360c76879f/480w.png thumbnail_dimensions: [480, 254] +channel: nightly --- diff --git a/examples/python/segment_anything_model/README.md b/examples/python/segment_anything_model/README.md index 287e36ef6d9b..4a423519ff49 100644 --- a/examples/python/segment_anything_model/README.md +++ b/examples/python/segment_anything_model/README.md @@ -4,6 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/segment_an tags: [2D, sam, segmentation] thumbnail: https://static.rerun.io/segment_anything_model/6aa2651907efbcf81be55b343caa76b9de5f2138/480w.png thumbnail_dimensions: [480, 283] +channel: nightly --- diff --git a/examples/python/structure_from_motion/README.md b/examples/python/structure_from_motion/README.md index 2e53a143dda9..6aa2079b3993 100644 --- a/examples/python/structure_from_motion/README.md +++ b/examples/python/structure_from_motion/README.md @@ -5,7 +5,7 @@ tags: [2D, 3D, colmap, pinhole-camera, time-series] description: "Visualize a sparse reconstruction by COLMAP, a general-purpose Structure-from-Motion and Multi-View Stereo pipeline." thumbnail: https://static.rerun.io/structure_from_motion/b17f8824291fa1102a4dc2184d13c91f92d2279c/480w.png thumbnail_dimensions: [480, 275] -demo: true +channel: main build_args: ["--dataset=colmap_fiat", "--resize=800x600"] --- diff --git a/examples/rust/dna/README.md b/examples/rust/dna/README.md index e6ef49761d52..84810c0c2fc0 100644 --- a/examples/rust/dna/README.md +++ b/examples/rust/dna/README.md @@ -7,7 +7,7 @@ tags: [3d, api-example] description: "Simple example of logging point and line primitives to draw a 3D helix." thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/pixi.toml b/pixi.toml index 2ba3364deda1..dcea8ee34379 100644 --- a/pixi.toml +++ b/pixi.toml @@ -28,7 +28,6 @@ version = "0.1.0" # codegen = "cargo run --package re_types_builder -- " build-examples = "cargo run -q --locked -p re_build_examples --" -build-examples-manifest = "cargo run -q --locked -p re_build_examples_manifest --" # Assorted linting tasks fast-lint = "python scripts/fast_lint.py" diff --git a/scripts/ci/requirements-web-demo.txt b/scripts/ci/requirements-examples-main.txt similarity index 69% rename from scripts/ci/requirements-web-demo.txt rename to scripts/ci/requirements-examples-main.txt index 4aef8e38e6ac..effe1f627360 100644 --- a/scripts/ci/requirements-web-demo.txt +++ b/scripts/ci/requirements-examples-main.txt @@ -1,12 +1,6 @@ -r ../../examples/python/arkit_scenes/requirements.txt --r ../../examples/python/detect_and_track_objects/requirements.txt -r ../../examples/python/dicom_mri/requirements.txt -r ../../examples/python/dna/requirements.txt -r ../../examples/python/human_pose_tracking/requirements.txt -r ../../examples/python/plots/requirements.txt -r ../../examples/python/structure_from_motion/requirements.txt - -python-frontmatter==1.0.0 -requests>=2.31,<3 -Jinja2==3.1.2 -Pillow==10.0.0 diff --git a/scripts/ci/requirements-examples-nightly.txt b/scripts/ci/requirements-examples-nightly.txt new file mode 100644 index 000000000000..e67f8d683062 --- /dev/null +++ b/scripts/ci/requirements-examples-nightly.txt @@ -0,0 +1,10 @@ +-r ./requirements-examples-main.txt + +-r ../../examples/python/depth_guided_stable_diffusion/requirements.txt +-r ../../examples/python/detect_and_track_objects/requirements.txt +-r ../../examples/python/nuscenes/requirements.txt +-r ../../examples/python/objectron/requirements.txt +-r ../../examples/python/open_photogrammetry_format/requirements.txt +-r ../../examples/python/raw_mesh/requirements.txt +-r ../../examples/python/rgbd/requirements.txt +-r ../../examples/python/segment_anything_model/requirements.txt diff --git a/scripts/screenshot_compare/build_screenshot_compare.py b/scripts/screenshot_compare/build_screenshot_compare.py index e2f039718f93..c62bd3016c46 100755 --- a/scripts/screenshot_compare/build_screenshot_compare.py +++ b/scripts/screenshot_compare/build_screenshot_compare.py @@ -151,11 +151,21 @@ def collect_code_examples() -> Iterable[Example]: def build_demo_examples() -> None: - cmd = ["cargo", "run", "--locked", "-p", "re_build_examples", "--", "example_data"] + # fmt: off + cmd = [ + "cargo", "run", "--locked", + "-p", "re_build_examples", "--", + "rrd", "example_data", + ] run(cmd, cwd=RERUN_DIR) - cmd = ["cargo", "run", "--locked", "-p", "re_build_examples_manifest", "--", "example_data/examples_manifest.json"] + cmd = [ + "cargo", "run", "--locked", + "-p", "re_build_examples", "--", + "manifest", "example_data/examples_manifest.json", + ] run(cmd, cwd=RERUN_DIR) + # fmt: on def collect_demo_examples() -> Iterable[Example]: