From 0ea3cea196c89269c05a994cb73a0852298ca0e2 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:16:32 +0100 Subject: [PATCH 01/45] rename `main` to `nightly` --- .github/pull_request_template.md | 2 +- .github/workflows/on_push_main.yml | 2 +- .github/workflows/reusable_build_web.yml | 2 +- .github/workflows/reusable_upload_examples.yml | 5 ++--- .github/workflows/reusable_upload_web.yml | 5 ++--- crates/re_build_examples_manifest/src/main.rs | 10 +++++----- crates/re_viewer/src/ui/welcome_screen/example_page.rs | 4 ++-- 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 30f5ea0ce162..7c68e4d28549 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -17,7 +17,7 @@ To get an auto-generated PR description you can put "copilot:summary" or "copilo * [ ] I've included a screenshot or gif (if applicable) * [ ] I have tested the web demo (if applicable): * Full build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html) - * Partial build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way + * Partial build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way * [ ] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }}) diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml index 4915956da4c8..10a866bb39ae 100644 --- a/.github/workflows/on_push_main.yml +++ b/.github/workflows/on_push_main.yml @@ -56,7 +56,7 @@ jobs: uses: ./.github/workflows/reusable_deploy_docs.yml with: CONCURRENCY: push-${{ github.ref_name }} - PY_AND_CPP_DOCS_VERSION_NAME: "nightly" + PY_AND_CPP_DOCS_VERSION_NAME: "main" UPDATE_LATEST: false secrets: inherit diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 584b9e57aca6..1047d45fe79d 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -60,7 +60,7 @@ jobs: cargo run --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }} # We build a single manifest pointing to the `commit` - # All the `pr`, `nightly`, release tag, etc. variants will always just point to the resolved commit + # All the `pr`, `main`, release tag, etc. variants will always just point to the resolved commit - name: Build examples manifest shell: bash run: | diff --git a/.github/workflows/reusable_upload_examples.yml b/.github/workflows/reusable_upload_examples.yml index c9b3550917be..514406c0435d 100644 --- a/.github/workflows/reusable_upload_examples.yml +++ b/.github/workflows/reusable_upload_examples.yml @@ -93,13 +93,12 @@ jobs: headers: |- cache-control: no-cache, max-age=0 - - name: "Upload examples (nightly)" - # TEMP: Tracking `main` until we can do actual nightly builds. + - name: "Upload examples (main)" if: github.ref == 'refs/heads/main' uses: google-github-actions/upload-cloud-storage@v1 with: path: "example_data" - destination: "rerun-web-viewer/version/nightly/examples" + destination: "rerun-web-viewer/version/main/examples" parent: false headers: |- cache-control: no-cache, max-age=0 diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index db29e82c37f4..f2b2fdf593d0 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -94,13 +94,12 @@ jobs: headers: |- cache-control: no-cache, max-age=0 - - name: "Upload web-viewer (nightly)" - # TEMP: Tracking `main` until we can do actual nightly builds. + - name: "Upload web-viewer (main)" if: github.ref == 'refs/heads/main' uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" - destination: "rerun-web-viewer/version/nightly" + destination: "rerun-web-viewer/version/main" parent: false headers: |- cache-control: no-cache, max-age=0 diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs index 310e756971e0..c16f85b17829 100644 --- a/crates/re_build_examples_manifest/src/main.rs +++ b/crates/re_build_examples_manifest/src/main.rs @@ -10,8 +10,8 @@ //! - On any `release-x.y.z` branch, the version is `version/x.y.z` //! - On any other branch, the version is `commit/$COMMIT_SHORT_HASH` //! -//! Otherwise, the version is `version/nightly`. This means local builds, -//! and builds on `main` point to `version/nightly`. +//! Otherwise, the version is `version/main`. This means local builds, +//! and builds on `main` point to `version/main`. use std::path::Path; use std::path::PathBuf; @@ -23,7 +23,7 @@ Usage: [options] [output_path] Options: -h, --help Print help - --base-url Where all examples are uploaded, e.g. `https://demo.rerun.io/version/nightly`. + --base-url Where all examples are uploaded, e.g. `https://app.rerun.io/version/main`. "; fn main() -> anyhow::Result<()> { @@ -228,10 +228,10 @@ fn get_base_url(build_env: Environment) -> anyhow::Result { let versioned_manifest = matches!(build_env, Environment::CondaBuild) || { let branch = re_build_tools::git_branch()?; if branch == "main" || !re_build_tools::is_on_ci() { - // on `main` and local builds, use `version/nightly` + // on `main` and local builds, use `version/main` // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` // on every commit to the `main` branch - return Ok("https://app.rerun.io/version/nightly".into()); + return Ok("https://app.rerun.io/version/main".into()); } parse_release_version(&branch).is_some() }; diff --git a/crates/re_viewer/src/ui/welcome_screen/example_page.rs b/crates/re_viewer/src/ui/welcome_screen/example_page.rs index 9fbfa1617362..3ee0c06fd753 100644 --- a/crates/re_viewer/src/ui/welcome_screen/example_page.rs +++ b/crates/re_viewer/src/ui/welcome_screen/example_page.rs @@ -127,10 +127,10 @@ pub(super) struct ExamplePage { fn default_manifest_url() -> String { let build_info = re_build_info::build_info!(); - // Always point to `version/nightly` for rerun devs, + // Always point to `version/main` for rerun devs, // because the current commit's manifest is unlikely to be uploaded to GCS. if build_info.is_in_rerun_workspace { - return "https://app.rerun.io/version/nightly/examples_manifest.json".into(); + return "https://app.rerun.io/version/main/examples_manifest.json".into(); } // Otherwise point to the current commit From 0596f974792744602085a26e1d9a805dc42c17ad Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:24:35 +0100 Subject: [PATCH 02/45] add nightly build --- .github/workflows/nightly.yml | 62 +++++++++++++++++++ .../workflows/reusable_upload_examples.yml | 14 +++++ .github/workflows/reusable_upload_web.yml | 14 +++++ 3 files changed, 90 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000000..734bbecd6496 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,62 @@ +name: Nightly + +on: + pull_request: + types: + - opened + - synchronize + # schedule: + # - cron: "0 12 * * *" + +jobs: + deploy-docs: + name: Deploy Docs + uses: ./.github/workflows/reusable_deploy_docs.yml + with: + CONCURRENCY: nightly + PY_AND_CPP_DOCS_VERSION_NAME: "nightly" + UPDATE_LATEST: false + secrets: inherit + + build-web: + name: "Build Web" + uses: ./.github/workflows/reusable_build_web.yml + with: + CONCURRENCY: nightly + secrets: inherit + + upload-web: + name: "Upload Web" + needs: [build-web] + uses: ./.github/workflows/reusable_upload_web.yml + with: + CONCURRENCY: nightly + NIGHTLY: true + secrets: inherit + + build-wheel-linux: + name: "Linux: Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: nightly-linux + PLATFORM: linux + WHEEL_ARTIFACT_NAME: linux-wheel + MODE: "pr" + secrets: inherit + + build-examples: + name: "Build Examples" + needs: [build-wheel-linux] + uses: ./.github/workflows/reusable_build_examples.yml + with: + CONCURRENCY: nightly + WHEEL_ARTIFACT_NAME: linux-wheel + secrets: inherit + + upload-examples: + name: "Upload Examples" + needs: [build-examples] + uses: ./.github/workflows/reusable_upload_examples.yml + with: + CONCURRENCY: nightly + secrets: inherit diff --git a/.github/workflows/reusable_upload_examples.yml b/.github/workflows/reusable_upload_examples.yml index 514406c0435d..436f251f8098 100644 --- a/.github/workflows/reusable_upload_examples.yml +++ b/.github/workflows/reusable_upload_examples.yml @@ -21,6 +21,10 @@ on: PR_NUMBER: type: string default: "" + NIGHTLY: + required: false + type: boolean + default: false concurrency: group: ${{ inputs.CONCURRENCY }}-upload-examples @@ -112,3 +116,13 @@ jobs: parent: false headers: |- cache-control: no-cache, max-age=0 + + - name: "Upload examples (nightly)" + if: ${{ inputs.NIGHTLY }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "example_data" + destination: "rerun-web-viewer/version/nightly/examples" + parent: false + headers: |- + cache-control: no-cache, max-age=0 diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index f2b2fdf593d0..615cb3bf6965 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -22,6 +22,10 @@ on: required: false type: string default: "" + NIGHTLY: + required: false + type: boolean + default: false concurrency: group: ${{ inputs.CONCURRENCY }}-upload-web @@ -113,3 +117,13 @@ jobs: parent: false headers: |- cache-control: no-cache, max-age=0 + + - name: "Upload web-viewer (nightly)" + if: ${{ inputs.NIGHTLY }} + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/version/nightly" + parent: false + headers: |- + cache-control: no-cache, max-age=0 From 5ec618298f6a777720355ad6fe3c549c2447695d Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:04:58 +0100 Subject: [PATCH 03/45] move common examples build logic into crate --- Cargo.lock | 12 ++ Cargo.toml | 1 + crates/re_build_examples/Cargo.toml | 1 + crates/re_build_examples/src/main.rs | 86 +------------ crates/re_build_examples_manifest/Cargo.toml | 1 + crates/re_build_examples_manifest/src/main.rs | 96 ++------------ crates/re_examples/Cargo.toml | 24 ++++ crates/re_examples/src/lib.rs | 117 ++++++++++++++++++ 8 files changed, 170 insertions(+), 168 deletions(-) create mode 100644 crates/re_examples/Cargo.toml create mode 100644 crates/re_examples/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 436f1e62c8dd..f2a8dc4ca49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4380,6 +4380,7 @@ dependencies = [ "indicatif", "rayon", "re_build_tools", + "re_examples", "serde", "serde_yaml", ] @@ -4390,6 +4391,7 @@ version = "0.12.0-alpha.1+dev" dependencies = [ "anyhow", "re_build_tools", + "re_examples", "serde", "serde_json", "serde_yaml", @@ -4518,6 +4520,16 @@ dependencies = [ "anyhow", ] +[[package]] +name = "re_examples" +version = "0.12.0-alpha.1+dev" +dependencies = [ + "anyhow", + "re_build_tools", + "serde", + "serde_yaml", +] + [[package]] name = "re_format" version = "0.12.0-alpha.1+dev" diff --git a/Cargo.toml b/Cargo.toml index a0aa614c79d9..71d1cead01db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ re_data_source = { path = "crates/re_data_source", version = "=0.12.0-alpha.1", re_data_store = { path = "crates/re_data_store", version = "=0.12.0-alpha.1", default-features = false } re_data_ui = { path = "crates/re_data_ui", version = "=0.12.0-alpha.1", default-features = false } re_error = { path = "crates/re_error", version = "=0.12.0-alpha.1", default-features = false } +re_examples = { path = "crates/re_examples", version = "=0.12.0-alpha.1", default-features = false } re_format = { path = "crates/re_format", version = "=0.12.0-alpha.1", default-features = false } re_int_histogram = { path = "crates/re_int_histogram", version = "=0.12.0-alpha.1", default-features = false } re_log = { path = "crates/re_log", version = "=0.12.0-alpha.1", default-features = false } diff --git a/crates/re_build_examples/Cargo.toml b/crates/re_build_examples/Cargo.toml index 674119cb72d8..4ddcf9462a6d 100644 --- a/crates/re_build_examples/Cargo.toml +++ b/crates/re_build_examples/Cargo.toml @@ -18,6 +18,7 @@ all-features = true [dependencies] re_build_tools.workspace = true +re_examples.workspace = true # External anyhow.workspace = true diff --git a/crates/re_build_examples/src/main.rs b/crates/re_build_examples/src/main.rs index 945320115902..3c156b950f19 100644 --- a/crates/re_build_examples/src/main.rs +++ b/crates/re_build_examples/src/main.rs @@ -7,8 +7,6 @@ //! `build_args` string array. use std::fs::create_dir_all; -use std::fs::read_dir; -use std::fs::read_to_string; use std::io::stdout; use std::io::IsTerminal; use std::path::Path; @@ -22,6 +20,7 @@ use indicatif::MultiProgress; use indicatif::ProgressBar; use rayon::prelude::IntoParallelIterator; use rayon::prelude::ParallelIterator; +use re_examples::{examples, Example}; const USAGE: &str = "\ Usage: [options] [output_dir] @@ -41,7 +40,7 @@ fn main() -> anyhow::Result<()> { let progress = MultiProgress::new(); let results: Vec> = examples .into_par_iter() - .map(|example| example.run(&progress, &args.output_dir)) + .map(|example| example.build(&progress, &args.output_dir)) .collect(); let mut failed = false; @@ -95,22 +94,12 @@ impl Args { } } -#[derive(serde::Deserialize)] -struct Frontmatter { - #[serde(default)] - demo: bool, - #[serde(default)] - build_args: Vec, +trait Build { + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result<()>; } -struct Example { - name: String, - script_path: PathBuf, - script_args: Vec, -} - -impl Example { - fn run(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result<()> { +impl Build for Example { + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result<()> { let mut cmd = Command::new("python3"); cmd.arg(self.script_path); cmd.arg("--save") @@ -174,66 +163,3 @@ fn wait_for_output( Ok(output) } - -fn examples() -> anyhow::Result> { - let mut examples = vec![]; - let dir = Path::new("examples/python"); - if !dir.exists() { - anyhow::bail!("Failed to find {}", dir.display()) - } - if !dir.is_dir() { - anyhow::bail!("{} is not a directory", dir.display()) - } - - for folder in read_dir(dir)? { - let folder = folder?; - let metadata = folder.metadata()?; - let name = folder.file_name().to_string_lossy().to_string(); - let readme = folder.path().join("README.md"); - if metadata.is_dir() && readme.exists() { - let readme = parse_frontmatter(readme)?; - if let Some(readme) = readme { - if readme.demo { - eprintln!("Adding example {name:?}"); - examples.push(Example { - name, - script_path: folder.path().join("main.py"), - script_args: readme.build_args, - }); - } else { - eprintln!("Skipping example {name:?} because 'demo' is set to 'false'"); - } - } else { - eprintln!("Skipping example {name:?} because it has no frontmatter"); - } - } - } - - if examples.is_empty() { - anyhow::bail!("No examples found in {}", dir.display()) - } - - examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); - Ok(examples) -} - -fn parse_frontmatter>(path: P) -> anyhow::Result> { - let path = path.as_ref(); - let content = read_to_string(path)?; - let content = content.replace('\r', ""); // Windows, god damn you - re_build_tools::rerun_if_changed(path); - let Some(content) = content.strip_prefix("---\n") else { - return Ok(None); - }; - let Some(end) = content.find("---") else { - anyhow::bail!("{:?} has invalid frontmatter", path); - }; - Ok(Some(serde_yaml::from_str(&content[..end]).map_err( - |e| { - anyhow::anyhow!( - "failed to read {:?}: {e}", - path.parent().unwrap().file_name().unwrap() - ) - }, - )?)) -} diff --git a/crates/re_build_examples_manifest/Cargo.toml b/crates/re_build_examples_manifest/Cargo.toml index ddc43a09e4a1..d3970c366033 100644 --- a/crates/re_build_examples_manifest/Cargo.toml +++ b/crates/re_build_examples_manifest/Cargo.toml @@ -18,6 +18,7 @@ all-features = true [dependencies] re_build_tools.workspace = true +re_examples.workspace = true # External anyhow.workspace = true diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs index c16f85b17829..f20d4e3204de 100644 --- a/crates/re_build_examples_manifest/src/main.rs +++ b/crates/re_build_examples_manifest/src/main.rs @@ -13,10 +13,10 @@ //! Otherwise, the version is `version/main`. This means local builds, //! and builds on `main` point to `version/main`. -use std::path::Path; use std::path::PathBuf; use re_build_tools::Environment; +use re_examples::{examples, Example}; const USAGE: &str = "\ Usage: [options] [output_path] @@ -106,22 +106,6 @@ fn build_examples_manifest(build_env: Environment, args: &Args) -> anyhow::Resul Ok(serde_json::to_string_pretty(&manifest)?) } -#[derive(serde::Deserialize)] -struct Frontmatter { - #[serde(default)] - title: String, - #[serde(default)] - tags: Vec, - #[serde(default)] - description: String, - #[serde(default)] - thumbnail: String, - #[serde(default)] - thumbnail_dimensions: [u64; 2], - #[serde(default)] - demo: bool, -} - #[derive(serde::Serialize)] struct ManifestEntry { name: String, @@ -134,16 +118,16 @@ struct ManifestEntry { impl ManifestEntry { fn new(example: Example, base_url: &str) -> Self { - let Example { name, readme } = example; + let name = example.name; Self { - title: readme.title, - description: readme.description, - tags: readme.tags, + title: example.title, + description: example.description, + tags: example.tags, rrd_url: format!("{base_url}/examples/{name}.rrd"), thumbnail: Thumbnail { - url: readme.thumbnail, - width: readme.thumbnail_dimensions[0], - height: readme.thumbnail_dimensions[1], + url: example.thumbnail_url, + width: example.thumbnail_dimensions[0], + height: example.thumbnail_dimensions[1], }, name, } @@ -157,70 +141,6 @@ struct Thumbnail { height: u64, } -struct Example { - name: String, - readme: Frontmatter, -} - -fn examples() -> anyhow::Result> { - let mut examples = vec![]; - let dir = Path::new("examples/python"); - if !dir.exists() { - anyhow::bail!("Failed to find {}", dir.display()) - } - if !dir.is_dir() { - anyhow::bail!("{} is not a directory", dir.display()) - } - - for folder in std::fs::read_dir(dir)? { - let folder = folder?; - let metadata = folder.metadata()?; - let name = folder.file_name().to_string_lossy().to_string(); - let readme = folder.path().join("README.md"); - if metadata.is_dir() && readme.exists() { - let readme = parse_frontmatter(readme)?; - if let Some(readme) = readme { - if readme.demo { - eprintln!("Adding example {name:?}"); - examples.push(Example { name, readme }); - } else { - eprintln!("Skipping example {name:?} because 'demo' is set to 'false'"); - } - } else { - eprintln!("Skipping example {name:?} because it has no frontmatter"); - } - } - } - - if examples.is_empty() { - anyhow::bail!("No examples found in {}", dir.display()) - } - - examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); - Ok(examples) -} - -fn parse_frontmatter>(path: P) -> anyhow::Result> { - let path = path.as_ref(); - let content = std::fs::read_to_string(path)?; - let content = content.replace('\r', ""); // Windows, god damn you - re_build_tools::rerun_if_changed(path); - let Some(content) = content.strip_prefix("---\n") else { - return Ok(None); - }; - let Some(end) = content.find("---") else { - anyhow::bail!("{:?} has invalid frontmatter", path); - }; - Ok(Some(serde_yaml::from_str(&content[..end]).map_err( - |e| { - anyhow::anyhow!( - "failed to read {:?}: {e}", - path.parent().unwrap().file_name().unwrap() - ) - }, - )?)) -} - fn get_base_url(build_env: Environment) -> anyhow::Result { // In the CondaBuild environment we can't trust the git_branch name -- if it exists // at all it's going to be the feedstock branch-name, not our Rerun branch. However diff --git a/crates/re_examples/Cargo.toml b/crates/re_examples/Cargo.toml new file mode 100644 index 000000000000..416e0eb4a9ed --- /dev/null +++ b/crates/re_examples/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "re_examples" +authors.workspace = true +description = "Utils for rerun example builds" +edition.workspace = true +homepage.workspace = true +include.workspace = true +license.workspace = true +publish = false +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +re_build_tools.workspace = true + +# External +anyhow.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_yaml.workspace = true diff --git a/crates/re_examples/src/lib.rs b/crates/re_examples/src/lib.rs new file mode 100644 index 000000000000..fc9b1eccf29d --- /dev/null +++ b/crates/re_examples/src/lib.rs @@ -0,0 +1,117 @@ +use std::path::Path; +use std::path::PathBuf; + +pub struct Example { + pub name: String, + pub title: String, + pub description: String, + pub tags: Vec, + pub thumbnail_url: String, + pub thumbnail_dimensions: [u64; 2], + pub script_path: PathBuf, + pub script_args: Vec, + pub kind: ExampleKind, +} + +pub enum ExampleKind { + Ignore, + Demo, + Nightly, +} + +impl ExampleKind { + fn infer(demo: bool, nightly: bool) -> Self { + if demo && nightly { + Self::Nightly + } else if demo { + Self::Demo + } else { + Self::Ignore + } + } +} + +#[derive(serde::Deserialize)] +struct Frontmatter { + #[serde(default)] + title: String, + #[serde(default)] + tags: Vec, + #[serde(default)] + description: String, + #[serde(default)] + thumbnail: String, + #[serde(default)] + thumbnail_dimensions: [u64; 2], + #[serde(default)] + demo: bool, + #[serde(default)] + nightly: bool, + #[serde(default)] + build_args: Vec, +} + +pub fn examples() -> anyhow::Result> { + let mut examples = vec![]; + let dir = Path::new("examples/python"); + if !dir.exists() { + anyhow::bail!("Failed to find {}", dir.display()) + } + if !dir.is_dir() { + anyhow::bail!("{} is not a directory", dir.display()) + } + + for folder in std::fs::read_dir(dir)? { + let folder = folder?; + let metadata = folder.metadata()?; + let name = folder.file_name().to_string_lossy().to_string(); + let readme = folder.path().join("README.md"); + if metadata.is_dir() && readme.exists() { + let readme = parse_frontmatter(readme)?; + if let Some(readme) = readme { + eprintln!("Adding example {name:?}"); + examples.push(Example { + name, + title: readme.title, + description: readme.description, + tags: readme.tags, + thumbnail_url: readme.thumbnail, + thumbnail_dimensions: readme.thumbnail_dimensions, + script_path: folder.path().join("main.py"), + script_args: readme.build_args, + kind: ExampleKind::infer(readme.demo, readme.nightly), + }); + } else { + eprintln!("Skipping example {name:?} because it has no frontmatter"); + } + } + } + + if examples.is_empty() { + anyhow::bail!("No examples found in {}", dir.display()) + } + + examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(examples) +} + +fn parse_frontmatter>(path: P) -> anyhow::Result> { + let path = path.as_ref(); + let content = std::fs::read_to_string(path)?; + let content = content.replace('\r', ""); // Windows, god damn you + re_build_tools::rerun_if_changed(path); + let Some(content) = content.strip_prefix("---\n") else { + return Ok(None); + }; + let Some(end) = content.find("---") else { + anyhow::bail!("{:?} has invalid frontmatter", path); + }; + Ok(Some(serde_yaml::from_str(&content[..end]).map_err( + |e| { + anyhow::anyhow!( + "failed to read {:?}: {e}", + path.parent().unwrap().file_name().unwrap() + ) + }, + )?)) +} From ff1855260d01eea95e83eb9d0d069372d490f5de Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:37:58 +0100 Subject: [PATCH 04/45] support multiple channels --- crates/re_build_examples/src/main.rs | 34 ++++- crates/re_build_examples_manifest/src/main.rs | 33 ++++- crates/re_examples/src/lib.rs | 140 ++++++++++++------ 3 files changed, 150 insertions(+), 57 deletions(-) diff --git a/crates/re_build_examples/src/main.rs b/crates/re_build_examples/src/main.rs index 3c156b950f19..6a1531dc624a 100644 --- a/crates/re_build_examples/src/main.rs +++ b/crates/re_build_examples/src/main.rs @@ -20,13 +20,14 @@ use indicatif::MultiProgress; use indicatif::ProgressBar; use rayon::prelude::IntoParallelIterator; use rayon::prelude::ParallelIterator; -use re_examples::{examples, Example}; +use re_examples::{Channel, Example}; const USAGE: &str = "\ Usage: [options] [output_dir] Options: - -h, --help Print help + -h, --help Print help + --channel Determines which examples to build: `nightly`, `main` "; fn main() -> anyhow::Result<()> { @@ -36,7 +37,7 @@ fn main() -> anyhow::Result<()> { create_dir_all(&args.output_dir)?; - let examples = examples()?; + let examples = args.channel.examples()?; let progress = MultiProgress::new(); let results: Vec> = examples .into_par_iter() @@ -59,18 +60,34 @@ fn main() -> anyhow::Result<()> { struct Args { output_dir: PathBuf, + channel: Channel, } impl Args { fn from_env() -> Self { - let mut output_dir = None; + let mut output_dir: Option = None; + let mut channel: Option = None; - for arg in std::env::args().skip(1) { + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { match arg.as_str() { "--help" | "-h" => { println!("{USAGE}"); exit(1); } + "--channel" => { + let Some(value) = args.next() else { + eprintln!("Expected value after \"--channel\""); + println!("\n{USAGE}"); + std::process::exit(1); + }; + let Ok(value) = value.parse::() else { + eprintln!("Invalid \"--channel\", expected `nightly` or `main`"); + println!("\n{USAGE}"); + std::process::exit(1); + }; + channel = Some(value); + } _ if arg.starts_with('-') => { eprintln!("Unknown argument: {arg:?}"); println!("\n{USAGE}"); @@ -90,7 +107,12 @@ impl Args { exit(1); }; - Args { output_dir } + let channel = channel.unwrap_or_default(); + + Args { + output_dir, + channel, + } } } diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs index f20d4e3204de..3d83f752c07d 100644 --- a/crates/re_build_examples_manifest/src/main.rs +++ b/crates/re_build_examples_manifest/src/main.rs @@ -16,14 +16,15 @@ use std::path::PathBuf; use re_build_tools::Environment; -use re_examples::{examples, Example}; +use re_examples::{Channel, Example}; const USAGE: &str = "\ Usage: [options] [output_path] Options: -h, --help Print help - --base-url Where all examples are uploaded, e.g. `https://app.rerun.io/version/main`. + --base-url Where all examples are uploaded, e.g. `https://app.rerun.io/version/main` + --channel Determines which examples to build: `nightly`, `main` "; fn main() -> anyhow::Result<()> { @@ -40,12 +41,14 @@ fn main() -> anyhow::Result<()> { struct Args { output_path: PathBuf, base_url: Option, + channel: Channel, } impl Args { fn from_env() -> Self { - let mut output_path = None; - let mut base_url = None; + let mut output_path: Option = None; + let mut base_url: Option = None; + let mut channel: Option = None; let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { @@ -55,12 +58,25 @@ impl Args { std::process::exit(1); } "--base-url" => { - let Some(url) = args.next() else { + let Some(value) = args.next() else { eprintln!("Expected value after \"--base-url\""); println!("\n{USAGE}"); std::process::exit(1); }; - base_url = Some(url); + base_url = Some(value); + } + "--channel" => { + let Some(value) = args.next() else { + eprintln!("Expected value after \"--channel\""); + println!("\n{USAGE}"); + std::process::exit(1); + }; + let Ok(value) = value.parse::() else { + eprintln!("Invalid \"--channel\", expected `nightly` or `main`"); + println!("\n{USAGE}"); + std::process::exit(1); + }; + channel = Some(value); } _ if arg.starts_with('-') => { eprintln!("Unknown argument: {arg:?}"); @@ -81,9 +97,12 @@ impl Args { std::process::exit(1); }; + let channel = channel.unwrap_or_default(); + Args { output_path, base_url, + channel, } } } @@ -95,7 +114,7 @@ fn build_examples_manifest(build_env: Environment, args: &Args) -> anyhow::Resul }; let mut manifest = vec![]; - for example in examples()? { + for example in args.channel.examples()? { manifest.push(ManifestEntry::new(example, &base_url)); } diff --git a/crates/re_examples/src/lib.rs b/crates/re_examples/src/lib.rs index fc9b1eccf29d..3d3b47f0b198 100644 --- a/crates/re_examples/src/lib.rs +++ b/crates/re_examples/src/lib.rs @@ -1,5 +1,7 @@ +use std::fmt::Display; use std::path::Path; use std::path::PathBuf; +use std::str::FromStr; pub struct Example { pub name: String, @@ -13,6 +15,100 @@ pub struct Example { pub kind: ExampleKind, } +impl ExampleKind { + #[allow(clippy::match_like_matches_macro)] // harder to read + pub fn included_in(&self, channel: Channel) -> bool { + match (channel, self) { + (Channel::Nightly, ExampleKind::Nightly | ExampleKind::Demo) => true, + (Channel::Main, ExampleKind::Demo) => true, + _ => false, + } + } +} + +#[derive(Default, Clone, Copy)] +pub enum Channel { + #[default] + Main, + Nightly, +} + +impl Channel { + pub fn examples(self) -> anyhow::Result> { + let mut examples = vec![]; + let dir = Path::new("examples/python"); + if !dir.exists() { + anyhow::bail!("Failed to find {}", dir.display()) + } + if !dir.is_dir() { + anyhow::bail!("{} is not a directory", dir.display()) + } + + for folder in std::fs::read_dir(dir)? { + let folder = folder?; + let metadata = folder.metadata()?; + let name = folder.file_name().to_string_lossy().to_string(); + let readme = folder.path().join("README.md"); + if metadata.is_dir() && readme.exists() { + let readme = parse_frontmatter(readme)?; + let Some(readme) = readme else { + eprintln!("Skipping example {name:?} because it has no frontmatter"); + continue; + }; + + let kind = ExampleKind::infer(readme.demo, readme.nightly); + if !kind.included_in(self) { + eprintln!("Skipping example {name:?} because it is not included in the {self} channel"); + continue; + } + + eprintln!("Adding example {name:?}"); + examples.push(Example { + name, + title: readme.title, + description: readme.description, + tags: readme.tags, + thumbnail_url: readme.thumbnail, + thumbnail_dimensions: readme.thumbnail_dimensions, + script_path: folder.path().join("main.py"), + script_args: readme.build_args, + kind, + }); + } + } + + if examples.is_empty() { + anyhow::bail!("No examples found in {}", dir.display()) + } + + examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(examples) + } +} + +impl Display for Channel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + Channel::Main => "main", + Channel::Nightly => "nightly", + }; + f.write_str(s) + } +} + +impl FromStr for Channel { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "main" => Ok(Self::Main), + "nightly" => Ok(Self::Nightly), + _ => Err(()), + } + } +} + +#[derive(Clone, Copy)] pub enum ExampleKind { Ignore, Demo, @@ -51,50 +147,6 @@ struct Frontmatter { build_args: Vec, } -pub fn examples() -> anyhow::Result> { - let mut examples = vec![]; - let dir = Path::new("examples/python"); - if !dir.exists() { - anyhow::bail!("Failed to find {}", dir.display()) - } - if !dir.is_dir() { - anyhow::bail!("{} is not a directory", dir.display()) - } - - for folder in std::fs::read_dir(dir)? { - let folder = folder?; - let metadata = folder.metadata()?; - let name = folder.file_name().to_string_lossy().to_string(); - let readme = folder.path().join("README.md"); - if metadata.is_dir() && readme.exists() { - let readme = parse_frontmatter(readme)?; - if let Some(readme) = readme { - eprintln!("Adding example {name:?}"); - examples.push(Example { - name, - title: readme.title, - description: readme.description, - tags: readme.tags, - thumbnail_url: readme.thumbnail, - thumbnail_dimensions: readme.thumbnail_dimensions, - script_path: folder.path().join("main.py"), - script_args: readme.build_args, - kind: ExampleKind::infer(readme.demo, readme.nightly), - }); - } else { - eprintln!("Skipping example {name:?} because it has no frontmatter"); - } - } - } - - if examples.is_empty() { - anyhow::bail!("No examples found in {}", dir.display()) - } - - examples.sort_unstable_by(|a, b| a.name.cmp(&b.name)); - Ok(examples) -} - fn parse_frontmatter>(path: P) -> anyhow::Result> { let path = path.as_ref(); let content = std::fs::read_to_string(path)?; From 9f70cd1ec6c5facd8cd29cab4409cc3e205343ce Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:46:06 +0100 Subject: [PATCH 05/45] set channel --- .github/workflows/nightly.yml | 2 ++ .github/workflows/on_pull_request.yml | 2 ++ .github/workflows/on_push_main.yml | 2 ++ .github/workflows/reusable_build_examples.yml | 7 ++++++- .github/workflows/reusable_build_web.yml | 10 +++++++++- .github/workflows/reusable_publish_web.yml | 5 ++++- 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 734bbecd6496..1e554660fbf8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,6 +23,7 @@ jobs: uses: ./.github/workflows/reusable_build_web.yml with: CONCURRENCY: nightly + CHANNEL: nightly secrets: inherit upload-web: @@ -50,6 +51,7 @@ jobs: uses: ./.github/workflows/reusable_build_examples.yml with: CONCURRENCY: nightly + CHANNEL: nightly WHEEL_ARTIFACT_NAME: linux-wheel secrets: inherit diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index b68683339b48..3ca8d0e5703c 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -111,6 +111,7 @@ jobs: uses: ./.github/workflows/reusable_build_web.yml with: CONCURRENCY: pr-${{ github.event.pull_request.number }} + CHANNEL: main secrets: inherit upload-web: @@ -130,6 +131,7 @@ jobs: uses: ./.github/workflows/reusable_build_examples.yml with: CONCURRENCY: pr-${{ github.event.pull_request.number }} + CHANNEL: main WHEEL_ARTIFACT_NAME: linux-wheel-fast secrets: inherit diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml index 10a866bb39ae..61bd55a3e261 100644 --- a/.github/workflows/on_push_main.yml +++ b/.github/workflows/on_push_main.yml @@ -65,6 +65,7 @@ jobs: uses: ./.github/workflows/reusable_build_web.yml with: CONCURRENCY: push-${{ github.ref_name }} + CHANNEL: main secrets: inherit upload-web: @@ -81,6 +82,7 @@ jobs: uses: ./.github/workflows/reusable_build_examples.yml with: CONCURRENCY: push-${{ github.ref_name }} + CHANNEL: main WHEEL_ARTIFACT_NAME: linux-wheel secrets: inherit diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index 0bf2cce197c6..31e5b07ee8f6 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -9,6 +9,9 @@ on: WHEEL_ARTIFACT_NAME: required: true type: string + CHANNEL: # `nightly` or `main` + required: true + type: string concurrency: group: ${{ inputs.CONCURRENCY }}-build-examples @@ -84,7 +87,9 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples example_data + pixi run build-examples \ + --channel ${{ inputs.CHANNEL }} \ + example_data - name: Upload assets uses: actions/upload-artifact@v3 diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 1047d45fe79d..260f3bab04b3 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -14,6 +14,9 @@ on: required: false type: string default: "" + CHANNEL: # `nightly` or `main` + required: true + type: string concurrency: group: ${{ inputs.CONCURRENCY }}-build-web @@ -54,6 +57,10 @@ jobs: workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + - uses: prefix-dev/setup-pixi@v0.4.1 + with: + pixi-version: v0.6.0 + - name: Build web-viewer (release) shell: bash run: | @@ -67,8 +74,9 @@ jobs: full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" sha="$(echo $full_commit | cut -c1-7)" - cargo run --locked -p re_build_examples_manifest -- \ + pixi run build-examples-manifest \ --base-url "https://app.rerun.io/commit/$sha" \ + --channel "${{ inputs.CHANNEL }}" \ web_viewer/examples_manifest.json - name: Upload web assets diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml index b8884a5c4df2..fde68a3a8883 100644 --- a/.github/workflows/reusable_publish_web.yml +++ b/.github/workflows/reusable_publish_web.yml @@ -107,7 +107,9 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples web_viewer/examples + pixi run build-examples \ + --channel "nightly" \ + web_viewer/examples - name: Build examples manifest shell: bash @@ -117,6 +119,7 @@ jobs: pixi run build-examples-manifest \ --base-url "https://app.rerun.io/commit/$sha" \ + --channel "nightly" \ web_viewer/examples_manifest.json - name: Upload app.rerun.io (versioned) From b90195b138d8bbeb9ef1e06b1b599ec31c286b07 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:49:19 +0100 Subject: [PATCH 06/45] build `detect_and_track_objects` on nightly --- examples/python/detect_and_track_objects/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/python/detect_and_track_objects/README.md b/examples/python/detect_and_track_objects/README.md index 71a8effa0e0a..506f0feedaa1 100644 --- a/examples/python/detect_and_track_objects/README.md +++ b/examples/python/detect_and_track_objects/README.md @@ -6,6 +6,7 @@ description: "Visualize object detection and segmentation using the Huggingface thumbnail: https://static.rerun.io/detect_and_track_objects/59f5b97a8724f9037353409ab3d0b7cb47d1544b/480w.png thumbnail_dimensions: [480, 279] demo: true +nightly: true --- From 26d585c091f19f21d8290b6a1757f15c9cb7d376 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:27:38 +0100 Subject: [PATCH 07/45] build more examples --- .github/workflows/reusable_build_examples.yml | 6 +++++- .../python/depth_guided_stable_diffusion/README.md | 2 ++ examples/python/nuscenes/README.md | 2 ++ examples/python/objectron/README.md | 2 ++ examples/python/open_photogrammetry_format/README.md | 2 ++ examples/python/raw_mesh/README.md | 2 ++ examples/python/rgbd/README.md | 2 ++ examples/python/segment_anything_model/README.md | 2 ++ examples/python/signed_distance_fields/README.md | 2 ++ ...ts-web-demo.txt => requirements-examples-main.txt} | 6 ------ scripts/ci/requirements-examples-nightly.txt | 11 +++++++++++ 11 files changed, 32 insertions(+), 7 deletions(-) rename scripts/ci/{requirements-web-demo.txt => requirements-examples-main.txt} (69%) create mode 100644 scripts/ci/requirements-examples-nightly.txt diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index 31e5b07ee8f6..b037ed52e69b 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -62,10 +62,14 @@ jobs: with: pixi-version: v0.6.0 + - name: Install example dependencies + shell: bash + run: | + pixi run pip install -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt + - name: Install Python dependencies and wheel shell: bash run: | - pixi run pip install -r scripts/ci/requirements-web-demo.txt pixi run pip uninstall rerun-sdk -y pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 pixi run pip install rerun-sdk --no-index --find-links wheel diff --git a/examples/python/depth_guided_stable_diffusion/README.md b/examples/python/depth_guided_stable_diffusion/README.md index 9fc78a37491d..b3d802fdef11 100644 --- a/examples/python/depth_guided_stable_diffusion/README.md +++ b/examples/python/depth_guided_stable_diffusion/README.md @@ -4,6 +4,8 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/depth_guid tags: [2D, depth, huggingface, stable-diffusion, tensor, text] thumbnail: https://static.rerun.io/depth_guided_stable_diffusion/a85516aba09f72649517891d767e15383ce7f4ea/480w.png thumbnail_dimensions: [480, 253] +demo: true +nightly: true --- diff --git a/examples/python/nuscenes/README.md b/examples/python/nuscenes/README.md index ad1a7b6d4e5c..6139d108c330 100644 --- a/examples/python/nuscenes/README.md +++ b/examples/python/nuscenes/README.md @@ -5,6 +5,8 @@ tags: [lidar, 3D, 2D, object-detection, pinhole-camera] description: "Visualize the nuScenes dataset including lidar, radar, images, and bounding boxes." thumbnail: https://static.rerun.io/nuscenes/64a50a9d67cbb69ae872551989ee807b195f6b5d/480w.png thumbnail_dimensions: [480, 282] +demo: true +nightly: true --- diff --git a/examples/python/objectron/README.md b/examples/python/objectron/README.md index 77173cb5346c..4fd0406cabf8 100644 --- a/examples/python/objectron/README.md +++ b/examples/python/objectron/README.md @@ -5,6 +5,8 @@ rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/objectron/src/ tags: [2D, 3D, object-detection, pinhole-camera] thumbnail: https://static.rerun.io/objectron/8ea3a37e6b4af2e06f8e2ea5e70c1951af67fea8/480w.png thumbnail_dimensions: [480, 268] +demo: true +nightly: true --- diff --git a/examples/python/open_photogrammetry_format/README.md b/examples/python/open_photogrammetry_format/README.md index 87eaadb7b3d3..3c93a17466ab 100644 --- a/examples/python/open_photogrammetry_format/README.md +++ b/examples/python/open_photogrammetry_format/README.md @@ -4,6 +4,8 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/open_photo tags: [2d, 3d, camera, photogrammetry] thumbnail: https://static.rerun.io/open_photogrammetry_format/603d5605f9670889bc8bce3365f16b831fce1eb1/480w.png thumbnail_dimensions: [480, 310] +demo: true +nightly: true --- diff --git a/examples/python/raw_mesh/README.md b/examples/python/raw_mesh/README.md index f2b8bc16f453..ee85016f5f67 100644 --- a/examples/python/raw_mesh/README.md +++ b/examples/python/raw_mesh/README.md @@ -5,6 +5,8 @@ rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/raw_mesh/src/m tags: [mesh] thumbnail: https://static.rerun.io/raw_mesh/64bec98280b07794f7c9617f30ba2c20278601c3/480w.png thumbnail_dimensions: [480, 271] +demo: true +nightly: true --- diff --git a/examples/python/rgbd/README.md b/examples/python/rgbd/README.md index d2e2d219376b..248813098d63 100644 --- a/examples/python/rgbd/README.md +++ b/examples/python/rgbd/README.md @@ -4,6 +4,8 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/rgbd/main. tags: [2D, 3D, depth, nyud, pinhole-camera] thumbnail: https://static.rerun.io/rgbd/4109d29ed52fa0a8f980fcdd0e9673360c76879f/480w.png thumbnail_dimensions: [480, 254] +demo: true +nightly: true --- diff --git a/examples/python/segment_anything_model/README.md b/examples/python/segment_anything_model/README.md index 287e36ef6d9b..88907500b19d 100644 --- a/examples/python/segment_anything_model/README.md +++ b/examples/python/segment_anything_model/README.md @@ -4,6 +4,8 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/segment_an tags: [2D, sam, segmentation] thumbnail: https://static.rerun.io/segment_anything_model/6aa2651907efbcf81be55b343caa76b9de5f2138/480w.png thumbnail_dimensions: [480, 283] +demo: true +nightly: true --- diff --git a/examples/python/signed_distance_fields/README.md b/examples/python/signed_distance_fields/README.md index e6aba1383493..516c134b06bd 100644 --- a/examples/python/signed_distance_fields/README.md +++ b/examples/python/signed_distance_fields/README.md @@ -4,6 +4,8 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/signed_dis tags: [3D, mesh, tensor] thumbnail: https://static.rerun.io/signed_distance_fields/99f6a886ed6f41b6a8e9023ba917a98668eaee70/480w.png thumbnail_dimensions: [480, 294] +demo: true +nightly: true --- diff --git a/scripts/ci/requirements-web-demo.txt b/scripts/ci/requirements-examples-main.txt similarity index 69% rename from scripts/ci/requirements-web-demo.txt rename to scripts/ci/requirements-examples-main.txt index 4aef8e38e6ac..effe1f627360 100644 --- a/scripts/ci/requirements-web-demo.txt +++ b/scripts/ci/requirements-examples-main.txt @@ -1,12 +1,6 @@ -r ../../examples/python/arkit_scenes/requirements.txt --r ../../examples/python/detect_and_track_objects/requirements.txt -r ../../examples/python/dicom_mri/requirements.txt -r ../../examples/python/dna/requirements.txt -r ../../examples/python/human_pose_tracking/requirements.txt -r ../../examples/python/plots/requirements.txt -r ../../examples/python/structure_from_motion/requirements.txt - -python-frontmatter==1.0.0 -requests>=2.31,<3 -Jinja2==3.1.2 -Pillow==10.0.0 diff --git a/scripts/ci/requirements-examples-nightly.txt b/scripts/ci/requirements-examples-nightly.txt new file mode 100644 index 000000000000..b4fc02920dd8 --- /dev/null +++ b/scripts/ci/requirements-examples-nightly.txt @@ -0,0 +1,11 @@ +-r ./requirements-examples-main.txt + +-r ../../examples/python/depth_guided_stable_diffusion/requirements.txt +-r ../../examples/python/detect_and_track_objects/requirements.txt +-r ../../examples/python/nuscenes/requirements.txt +-r ../../examples/python/objectron/requirements.txt +-r ../../examples/python/open_photogrammetry_format/requirements.txt +-r ../../examples/python/raw_mesh/requirements.txt +-r ../../examples/python/rgbd/requirements.txt +-r ../../examples/python/segment_anything_model/requirements.txt +-r ../../examples/python/signed_distance_fields/requirements.txt From f9069f931299d5a81394ff391204c6f9c84a28d2 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:34:00 +0100 Subject: [PATCH 08/45] fix lint --- crates/re_examples/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/re_examples/src/lib.rs b/crates/re_examples/src/lib.rs index 3d3b47f0b198..026f616cd029 100644 --- a/crates/re_examples/src/lib.rs +++ b/crates/re_examples/src/lib.rs @@ -1,3 +1,5 @@ +//! Example collection and parsing. + use std::fmt::Display; use std::path::Path; use std::path::PathBuf; From f891f79920fd2243e960f3c5faf232481743eba8 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:34:06 +0100 Subject: [PATCH 09/45] rename jobs + add `workflow_dispatch` trigger --- .github/workflows/nightly.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1e554660fbf8..58ef78bc9b96 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,6 +1,7 @@ name: Nightly on: + workflow_dispatch: pull_request: types: - opened @@ -36,7 +37,7 @@ jobs: secrets: inherit build-wheel-linux: - name: "Linux: Build & Upload Wheels" + name: "Build & Upload Wheels" uses: ./.github/workflows/reusable_build_and_upload_wheels.yml with: CONCURRENCY: nightly-linux From e07287c4650de7eae544744a329657892cdcf5a2 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 12:43:11 +0100 Subject: [PATCH 10/45] add re_examples to `ARCHITECTURE.md` --- ARCHITECTURE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 20838b816bdb..efe010ac8df0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -177,6 +177,7 @@ Update instructions: |----------------------------|---------------------------------------------------------------| | re_build_info | Information about the build. Use together with re_build_tools | | re_build_tools | build.rs helpers for generating build info | +| re_examples | Utils for rerun example builds | | re_types_builder | Generates code for Rerun's SDKs from flatbuffers definitions. | | re_build_examples | Build rerun example RRD files | | re_build_examples_manifest | Build the rerun examples manifest JSON file | From 3f4fb1f77010a4d74ce6a1388d5cf351060e24b8 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:15:16 +0100 Subject: [PATCH 11/45] remove stable diffusion --- examples/python/depth_guided_stable_diffusion/README.md | 2 -- scripts/ci/requirements-examples-nightly.txt | 1 - src/mesh-to-sdf | 1 + src/segment-anything | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) create mode 160000 src/mesh-to-sdf create mode 160000 src/segment-anything diff --git a/examples/python/depth_guided_stable_diffusion/README.md b/examples/python/depth_guided_stable_diffusion/README.md index b3d802fdef11..9fc78a37491d 100644 --- a/examples/python/depth_guided_stable_diffusion/README.md +++ b/examples/python/depth_guided_stable_diffusion/README.md @@ -4,8 +4,6 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/depth_guid tags: [2D, depth, huggingface, stable-diffusion, tensor, text] thumbnail: https://static.rerun.io/depth_guided_stable_diffusion/a85516aba09f72649517891d767e15383ce7f4ea/480w.png thumbnail_dimensions: [480, 253] -demo: true -nightly: true --- diff --git a/scripts/ci/requirements-examples-nightly.txt b/scripts/ci/requirements-examples-nightly.txt index b4fc02920dd8..c58f80b56874 100644 --- a/scripts/ci/requirements-examples-nightly.txt +++ b/scripts/ci/requirements-examples-nightly.txt @@ -1,6 +1,5 @@ -r ./requirements-examples-main.txt --r ../../examples/python/depth_guided_stable_diffusion/requirements.txt -r ../../examples/python/detect_and_track_objects/requirements.txt -r ../../examples/python/nuscenes/requirements.txt -r ../../examples/python/objectron/requirements.txt diff --git a/src/mesh-to-sdf b/src/mesh-to-sdf new file mode 160000 index 000000000000..66036a747e82 --- /dev/null +++ b/src/mesh-to-sdf @@ -0,0 +1 @@ +Subproject commit 66036a747e82e7129f6afc74c5325d676a322114 diff --git a/src/segment-anything b/src/segment-anything new file mode 160000 index 000000000000..6fdee8f2727f --- /dev/null +++ b/src/segment-anything @@ -0,0 +1 @@ +Subproject commit 6fdee8f2727f4506cfbbe553e23b895e27956588 From 73d439adef380cffc46a736eb638ea75b8e9ff66 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:16:49 +0100 Subject: [PATCH 12/45] remove signed distance fields --- examples/python/signed_distance_fields/README.md | 2 -- scripts/ci/requirements-examples-nightly.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/python/signed_distance_fields/README.md b/examples/python/signed_distance_fields/README.md index 516c134b06bd..e6aba1383493 100644 --- a/examples/python/signed_distance_fields/README.md +++ b/examples/python/signed_distance_fields/README.md @@ -4,8 +4,6 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/signed_dis tags: [3D, mesh, tensor] thumbnail: https://static.rerun.io/signed_distance_fields/99f6a886ed6f41b6a8e9023ba917a98668eaee70/480w.png thumbnail_dimensions: [480, 294] -demo: true -nightly: true --- diff --git a/scripts/ci/requirements-examples-nightly.txt b/scripts/ci/requirements-examples-nightly.txt index c58f80b56874..e67f8d683062 100644 --- a/scripts/ci/requirements-examples-nightly.txt +++ b/scripts/ci/requirements-examples-nightly.txt @@ -1,5 +1,6 @@ -r ./requirements-examples-main.txt +-r ../../examples/python/depth_guided_stable_diffusion/requirements.txt -r ../../examples/python/detect_and_track_objects/requirements.txt -r ../../examples/python/nuscenes/requirements.txt -r ../../examples/python/objectron/requirements.txt @@ -7,4 +8,3 @@ -r ../../examples/python/raw_mesh/requirements.txt -r ../../examples/python/rgbd/requirements.txt -r ../../examples/python/segment_anything_model/requirements.txt --r ../../examples/python/signed_distance_fields/requirements.txt From 3eb8329d3ba900eb96cd2193bb4bb8205f0971f4 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:21:35 +0100 Subject: [PATCH 13/45] fix version requirement in depth guided stable diffusion + add it back --- examples/python/depth_guided_stable_diffusion/README.md | 2 ++ examples/python/depth_guided_stable_diffusion/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/python/depth_guided_stable_diffusion/README.md b/examples/python/depth_guided_stable_diffusion/README.md index 9fc78a37491d..b3d802fdef11 100644 --- a/examples/python/depth_guided_stable_diffusion/README.md +++ b/examples/python/depth_guided_stable_diffusion/README.md @@ -4,6 +4,8 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/depth_guid tags: [2D, depth, huggingface, stable-diffusion, tensor, text] thumbnail: https://static.rerun.io/depth_guided_stable_diffusion/a85516aba09f72649517891d767e15383ce7f4ea/480w.png thumbnail_dimensions: [480, 253] +demo: true +nightly: true --- diff --git a/examples/python/depth_guided_stable_diffusion/requirements.txt b/examples/python/depth_guided_stable_diffusion/requirements.txt index 184d26dd6ab6..642f1c5b000e 100644 --- a/examples/python/depth_guided_stable_diffusion/requirements.txt +++ b/examples/python/depth_guided_stable_diffusion/requirements.txt @@ -1,5 +1,5 @@ accelerate -diffusers>=0.12.1 +diffusers>=0.12.1,<=0.21 ftfy numpy packaging From 3c6abba4900364620738b4151d7d8fdec31a26dc Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:52:40 +0100 Subject: [PATCH 14/45] use `main` channel in publish web --- .github/workflows/reusable_publish_web.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml index fde68a3a8883..1e7450285c60 100644 --- a/.github/workflows/reusable_publish_web.yml +++ b/.github/workflows/reusable_publish_web.yml @@ -108,7 +108,7 @@ jobs: shell: bash run: | pixi run build-examples \ - --channel "nightly" \ + --channel "main" \ web_viewer/examples - name: Build examples manifest @@ -119,7 +119,7 @@ jobs: pixi run build-examples-manifest \ --base-url "https://app.rerun.io/commit/$sha" \ - --channel "nightly" \ + --channel "main" \ web_viewer/examples_manifest.json - name: Upload app.rerun.io (versioned) From 5125bd520d4a3ccd98232811356955339618a49e Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:53:21 +0100 Subject: [PATCH 15/45] don't deploy docs on nightly --- .github/workflows/nightly.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 58ef78bc9b96..40c5a683e72f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -10,15 +10,6 @@ on: # - cron: "0 12 * * *" jobs: - deploy-docs: - name: Deploy Docs - uses: ./.github/workflows/reusable_deploy_docs.yml - with: - CONCURRENCY: nightly - PY_AND_CPP_DOCS_VERSION_NAME: "nightly" - UPDATE_LATEST: false - secrets: inherit - build-web: name: "Build Web" uses: ./.github/workflows/reusable_build_web.yml From a396ab2218867575d5b440167c10c82da6e20bc9 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:56:21 +0100 Subject: [PATCH 16/45] don't use pixi --- .github/workflows/reusable_build_examples.yml | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index b037ed52e69b..ec71d47e4beb 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -58,28 +58,24 @@ jobs: name: ${{ inputs.WHEEL_ARTIFACT_NAME }} path: wheel - - uses: prefix-dev/setup-pixi@v0.4.1 - with: - pixi-version: v0.6.0 - - name: Install example dependencies shell: bash run: | - pixi run pip install -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt + pip install -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt - name: Install Python dependencies and wheel shell: bash run: | - pixi run pip uninstall rerun-sdk -y - pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 - pixi run pip install rerun-sdk --no-index --find-links wheel + pip uninstall rerun-sdk -y + pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 + pip install rerun-sdk --no-index --find-links wheel - name: Verify built wheel version shell: bash run: | - pixi run python3 -m rerun --version - pixi run which rerun - pixi run rerun --version + python3 -m rerun --version + which rerun + rerun --version - name: Get sha id: get-sha @@ -91,7 +87,7 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples \ + cargo run -q --locked -p re_build_examples -- \ --channel ${{ inputs.CHANNEL }} \ example_data From 1901a334b81ee863f09487dfb9cd957f19119933 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:04:15 +0100 Subject: [PATCH 17/45] `--no-input` when building examples --- .github/workflows/reusable_build_examples.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index ec71d47e4beb..9692a7c1895b 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -61,7 +61,9 @@ jobs: - name: Install example dependencies shell: bash run: | - pip install -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt + pip install \ + -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt \ + --no-input - name: Install Python dependencies and wheel shell: bash From deba0fe8de9270e0b4ea719d8d8a31b93ea33aa0 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:24:51 +0100 Subject: [PATCH 18/45] remove `mesh-to-sdf` and `segment-anything` from the repo --- src/mesh-to-sdf | 1 - src/segment-anything | 1 - 2 files changed, 2 deletions(-) delete mode 160000 src/mesh-to-sdf delete mode 160000 src/segment-anything diff --git a/src/mesh-to-sdf b/src/mesh-to-sdf deleted file mode 160000 index 66036a747e82..000000000000 --- a/src/mesh-to-sdf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 66036a747e82e7129f6afc74c5325d676a322114 diff --git a/src/segment-anything b/src/segment-anything deleted file mode 160000 index 6fdee8f2727f..000000000000 --- a/src/segment-anything +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6fdee8f2727f4506cfbbe553e23b895e27956588 From 4794b5054b4f570795c2f2636494fe5a6b9ac3c8 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:40:00 +0100 Subject: [PATCH 19/45] remove docker container + use pixi --- .github/workflows/reusable_build_examples.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index 9692a7c1895b..72dc05af9e36 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -18,7 +18,6 @@ concurrency: cancel-in-progress: true env: - PYTHON_VERSION: "3.8" # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html @@ -36,9 +35,6 @@ jobs: runs-on: ubuntu-latest-16-cores - container: - image: rerunio/ci_docker:0.11.0 - steps: - uses: actions/checkout@v4 with: @@ -52,6 +48,10 @@ jobs: workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + - uses: prefix-dev/setup-pixi@v0.4.1 + with: + pixi-version: v0.6.0 + - name: Download Wheel uses: actions/download-artifact@v3 with: @@ -61,23 +61,23 @@ jobs: - name: Install example dependencies shell: bash run: | - pip install \ + pixi run pip install \ -r scripts/ci/requirements-examples-${{ inputs.CHANNEL }}.txt \ --no-input - name: Install Python dependencies and wheel shell: bash run: | - pip uninstall rerun-sdk -y - pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 - pip install rerun-sdk --no-index --find-links wheel + pixi run pip uninstall rerun-sdk -y + pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 + pixi run pip install rerun-sdk --no-index --find-links wheel - name: Verify built wheel version shell: bash run: | - python3 -m rerun --version - which rerun - rerun --version + pixi run python3 -m rerun --version + pixi run which rerun + pixi run rerun --version - name: Get sha id: get-sha @@ -89,7 +89,7 @@ jobs: - name: Build examples shell: bash run: | - cargo run -q --locked -p re_build_examples -- \ + pixi run build-examples \ --channel ${{ inputs.CHANNEL }} \ example_data From 96099faf9fd1893f2296b11f0c6e2c8bd593b9af Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:29:57 +0100 Subject: [PATCH 20/45] set manifest urls properly --- crates/re_build_examples_manifest/src/main.rs | 8 ++++++-- crates/re_viewer/src/ui/welcome_screen/example_page.rs | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs index 3d83f752c07d..63b4c7bb3b2a 100644 --- a/crates/re_build_examples_manifest/src/main.rs +++ b/crates/re_build_examples_manifest/src/main.rs @@ -110,7 +110,7 @@ impl Args { fn build_examples_manifest(build_env: Environment, args: &Args) -> anyhow::Result { let base_url = match &args.base_url { Some(base_url) => base_url.clone(), - None => get_base_url(build_env)?, + None => get_base_url(build_env, args.channel)?, }; let mut manifest = vec![]; @@ -160,7 +160,11 @@ struct Thumbnail { height: u64, } -fn get_base_url(build_env: Environment) -> anyhow::Result { +fn get_base_url(build_env: Environment, channel: Channel) -> anyhow::Result { + if matches!(channel, Channel::Nightly) { + return Ok("https://app.rerun.io/version/nightly".into()); + } + // In the CondaBuild environment we can't trust the git_branch name -- if it exists // at all it's going to be the feedstock branch-name, not our Rerun branch. However // conda should ONLY be building released versions, so we want to version the manifest. diff --git a/crates/re_viewer/src/ui/welcome_screen/example_page.rs b/crates/re_viewer/src/ui/welcome_screen/example_page.rs index 3ee0c06fd753..f145d5ab3832 100644 --- a/crates/re_viewer/src/ui/welcome_screen/example_page.rs +++ b/crates/re_viewer/src/ui/welcome_screen/example_page.rs @@ -125,6 +125,11 @@ pub(super) struct ExamplePage { } fn default_manifest_url() -> String { + // Sometimes we want the default to point somewhere else, such as when doing nightly builds. + if let Some(url) = option_env!("DEFAULT_EXAMPLES_MANIFEST_URL") { + return url.into(); + } + let build_info = re_build_info::build_info!(); // Always point to `version/main` for rerun devs, From e5e1abf5f15b09ff79d7c03c98b37f8a31802ab1 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:30:06 +0100 Subject: [PATCH 21/45] dont upload commits on nightly build --- .github/workflows/reusable_upload_examples.yml | 1 + .github/workflows/reusable_upload_web.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/reusable_upload_examples.yml b/.github/workflows/reusable_upload_examples.yml index 436f251f8098..cfbd28f8e29c 100644 --- a/.github/workflows/reusable_upload_examples.yml +++ b/.github/workflows/reusable_upload_examples.yml @@ -65,6 +65,7 @@ jobs: echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: "Upload examples (commit)" + if: ${{ !inputs.NIGHTLY }} uses: google-github-actions/upload-cloud-storage@v1 with: path: "example_data" diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml index 615cb3bf6965..b2a43a745611 100644 --- a/.github/workflows/reusable_upload_web.yml +++ b/.github/workflows/reusable_upload_web.yml @@ -66,6 +66,7 @@ jobs: echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT" - name: "Upload web-viewer (commit)" + if: ${{ !inputs.NIGHTLY }} uses: google-github-actions/upload-cloud-storage@v1 with: path: "web_viewer" From 6d7ac40647998849e4e565c2975c21b23c5445db Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:30:31 +0100 Subject: [PATCH 22/45] set default manifest url in nightly build --- .github/workflows/reusable_build_web.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 260f3bab04b3..8484ce005e36 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -63,6 +63,8 @@ jobs: - name: Build web-viewer (release) shell: bash + env: + DEFAULT_EXAMPLES_MANIFEST_URL: "https://app.rerun.io/version/nightly/examples_manifest.json" run: | cargo run --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }} From cfc9e5afe3bd85a3254c89a611b1a555b7322e33 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:30:44 +0100 Subject: [PATCH 23/45] temp --- .github/workflows/nightly.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 40c5a683e72f..4cf90723a45d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,8 +6,9 @@ on: types: - opened - synchronize - # schedule: - # - cron: "0 12 * * *" + #schedule: + # # 12:15 every day + # - cron: "15 12 * * *" jobs: build-web: From 2662b0c6100892ae26013bd66d24f197a2f3a301 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:51:08 +0100 Subject: [PATCH 24/45] override url to `version/nightly` on nightly channel --- crates/re_build_examples_manifest/src/main.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs index 63b4c7bb3b2a..ec4741c23367 100644 --- a/crates/re_build_examples_manifest/src/main.rs +++ b/crates/re_build_examples_manifest/src/main.rs @@ -108,9 +108,13 @@ impl Args { } fn build_examples_manifest(build_env: Environment, args: &Args) -> anyhow::Result { - let base_url = match &args.base_url { - Some(base_url) => base_url.clone(), - None => get_base_url(build_env, args.channel)?, + let base_url = if matches!(args.channel, Channel::Nightly) { + "https://app.rerun.io/version/nightly".to_owned() + } else { + match &args.base_url { + Some(base_url) => base_url.clone(), + None => get_base_url(build_env)?, + } }; let mut manifest = vec![]; @@ -160,11 +164,7 @@ struct Thumbnail { height: u64, } -fn get_base_url(build_env: Environment, channel: Channel) -> anyhow::Result { - if matches!(channel, Channel::Nightly) { - return Ok("https://app.rerun.io/version/nightly".into()); - } - +fn get_base_url(build_env: Environment) -> anyhow::Result { // In the CondaBuild environment we can't trust the git_branch name -- if it exists // at all it's going to be the feedstock branch-name, not our Rerun branch. However // conda should ONLY be building released versions, so we want to version the manifest. From 7dd2c58c7b9cc4d1386fdc917aa6a3abefde6207 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:10:31 +0100 Subject: [PATCH 25/45] upload nightly --- .github/workflows/nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4cf90723a45d..42260eca16ec 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -54,4 +54,5 @@ jobs: uses: ./.github/workflows/reusable_upload_examples.yml with: CONCURRENCY: nightly + NIGHTLY: true secrets: inherit From 412e005169332c5be9e2ba5676ef804bf1bac351 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:46:10 +0100 Subject: [PATCH 26/45] set schedule --- .github/workflows/nightly.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 42260eca16ec..d14c7e9156b1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,13 +2,13 @@ name: Nightly on: workflow_dispatch: - pull_request: - types: - - opened - - synchronize - #schedule: - # # 12:15 every day - # - cron: "15 12 * * *" + # pull_request: + # types: + # - opened + # - synchronize + schedule: + # 12:15 every day + - cron: "15 12 * * *" jobs: build-web: From 6ab8cfab65388de1eb8512a8079b174864e445ae Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 14 Dec 2023 12:09:19 +0100 Subject: [PATCH 27/45] Print the final list of generated .rrd files and their sizes --- crates/re_build_examples/src/main.rs | 38 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/crates/re_build_examples/src/main.rs b/crates/re_build_examples/src/main.rs index 6a1531dc624a..74cc8db60a8b 100644 --- a/crates/re_build_examples/src/main.rs +++ b/crates/re_build_examples/src/main.rs @@ -39,16 +39,30 @@ fn main() -> anyhow::Result<()> { let examples = args.channel.examples()?; let progress = MultiProgress::new(); - let results: Vec> = examples + let results: Vec> = examples .into_par_iter() .map(|example| example.build(&progress, &args.output_dir)) .collect(); let mut failed = false; for result in results { - if let Err(err) = result { - eprintln!("{err}"); - failed = true; + match result { + Ok(rrd_path) => { + if let Ok(metadata) = std::fs::metadata(&rrd_path) { + println!( + "Output: {} ({})", + rrd_path.display(), + re_format::format_bytes(metadata.len() as _) + ); + } else { + eprintln!("Missing rrd at {}", rrd_path.display()); + failed = true; + } + } + Err(err) => { + eprintln!("{err}"); + failed = true; + } } } if failed { @@ -117,15 +131,17 @@ impl Args { } trait Build { - fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result<()>; + /// Returns the path to the resulting `.rrd` file. + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result; } impl Build for Example { - fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result<()> { + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result { + let rrd_path = output_dir.join(&self.name).with_extension("rrd"); + let mut cmd = Command::new("python3"); cmd.arg(self.script_path); - cmd.arg("--save") - .arg(output_dir.join(&self.name).with_extension("rrd")); + cmd.arg("--save").arg(&rrd_path); cmd.args(self.script_args); let final_args = cmd @@ -141,7 +157,9 @@ impl Build for Example { let output = wait_for_output(cmd, &self.name, progress)?; - if !output.status.success() { + if output.status.success() { + Ok(rrd_path) + } else { anyhow::bail!( "Failed to run `python3 {}`: \ \nstdout: \ @@ -153,8 +171,6 @@ impl Build for Example { String::from_utf8(output.stderr)?, ); } - - Ok(()) } } From 2a4a519c6d0fb63365faba16ed06fa064bcb2811 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 14 Dec 2023 12:09:46 +0100 Subject: [PATCH 28/45] Improve initial printout of example candidates --- crates/re_examples/src/lib.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/re_examples/src/lib.rs b/crates/re_examples/src/lib.rs index 026f616cd029..9bbb1a2b609d 100644 --- a/crates/re_examples/src/lib.rs +++ b/crates/re_examples/src/lib.rs @@ -46,25 +46,32 @@ impl Channel { anyhow::bail!("{} is not a directory", dir.display()) } - for folder in std::fs::read_dir(dir)? { - let folder = folder?; + let folders: std::collections::BTreeMap = + std::fs::read_dir(dir)? + .filter_map(Result::ok) + .map(|folder| { + let name = folder.file_name().to_string_lossy().to_string(); + (name, folder) + }) + .collect(); + + for (name, folder) in folders { let metadata = folder.metadata()?; - let name = folder.file_name().to_string_lossy().to_string(); let readme = folder.path().join("README.md"); if metadata.is_dir() && readme.exists() { let readme = parse_frontmatter(readme)?; let Some(readme) = readme else { - eprintln!("Skipping example {name:?} because it has no frontmatter"); + eprintln!("{name:?}: skipped - MISSING FRONTMATTER"); continue; }; let kind = ExampleKind::infer(readme.demo, readme.nightly); if !kind.included_in(self) { - eprintln!("Skipping example {name:?} because it is not included in the {self} channel"); + eprintln!("{name:?}: skipped"); continue; } - eprintln!("Adding example {name:?}"); + eprintln!("{name:?}: added"); examples.push(Example { name, title: readme.title, From cf2e509d89f62f33375f189d90fa2255946578e5 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 14 Dec 2023 12:10:11 +0100 Subject: [PATCH 29/45] Add missing dependency --- Cargo.lock | 1 + crates/re_build_examples/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 71edefd6f925..8bf77cd327eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4391,6 +4391,7 @@ dependencies = [ "rayon", "re_build_tools", "re_examples", + "re_format", "serde", "serde_yaml", ] diff --git a/crates/re_build_examples/Cargo.toml b/crates/re_build_examples/Cargo.toml index 4ddcf9462a6d..e3bc9937f1e6 100644 --- a/crates/re_build_examples/Cargo.toml +++ b/crates/re_build_examples/Cargo.toml @@ -19,6 +19,7 @@ all-features = true [dependencies] re_build_tools.workspace = true re_examples.workspace = true +re_format.workspace = true # External anyhow.workspace = true From cbd074455d26184613fb3b8bd9d459ed61386c1b Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:58:48 +0100 Subject: [PATCH 30/45] update pr template --- .github/pull_request_template.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7c68e4d28549..a8de771ae5ca 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,8 +16,9 @@ To get an auto-generated PR description you can put "copilot:summary" or "copilo * [ ] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [ ] I've included a screenshot or gif (if applicable) * [ ] I have tested the web demo (if applicable): - * Full build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html) - * Partial build: [app.rerun.io](https://app.rerun.io/pr/{{ pr.number }}/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) - Useful for quick testing when changes do not affect examples in any way + * Using newly built examples: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html) + * Using examples from latest `main` build: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) + * Using full set of examples from `nightly` build: [app.rerun.io](https://app.rerun.io/pr/{{pr.number}}/index.html?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [ ] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/{{ pr.number }}) From 004f79fb5cf2bdc5855379c503c4c67f54435575 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:01:06 +0100 Subject: [PATCH 31/45] clarify what timezone schedule is in --- .github/workflows/nightly.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d14c7e9156b1..51e4ad64d56f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,12 +2,9 @@ name: Nightly on: workflow_dispatch: - # pull_request: - # types: - # - opened - # - synchronize schedule: - # 12:15 every day + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule + # 12:15 UTC, every day - cron: "15 12 * * *" jobs: From d2a4a72b4baeda12d0fdf9fc25ea71a96b638157 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:45:41 +0100 Subject: [PATCH 32/45] merge example builds --- Cargo.lock | 54 +++-- Cargo.toml | 3 +- crates/re_build_examples/Cargo.toml | 3 +- .../src/example.rs} | 54 ++--- crates/re_build_examples/src/main.rs | 213 ++--------------- crates/re_build_examples/src/manifest.rs | 149 ++++++++++++ crates/re_build_examples/src/rrd.rs | 136 +++++++++++ crates/re_build_examples_manifest/Cargo.toml | 27 --- crates/re_build_examples_manifest/src/main.rs | 225 ------------------ crates/re_examples/Cargo.toml | 24 -- 10 files changed, 364 insertions(+), 524 deletions(-) rename crates/{re_examples/src/lib.rs => re_build_examples/src/example.rs} (80%) create mode 100644 crates/re_build_examples/src/manifest.rs create mode 100644 crates/re_build_examples/src/rrd.rs delete mode 100644 crates/re_build_examples_manifest/Cargo.toml delete mode 100644 crates/re_build_examples_manifest/src/main.rs delete mode 100644 crates/re_examples/Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index 8bf77cd327eb..ed5ac4cac822 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,6 +256,37 @@ dependencies = [ "x11rb", ] +[[package]] +name = "argh" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af5ba06967ff7214ce4c7419c7d185be7ecd6cc4965a8f6e1d8ce0398aad219" +dependencies = [ + "argh_derive", + "argh_shared", +] + +[[package]] +name = "argh_derive" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56df0aeedf6b7a2fc67d06db35b09684c3e8da0c95f8f27685cb17e08413d87a" +dependencies = [ + "argh_shared", + "proc-macro2", + "quote", + "syn 2.0.32", +] + +[[package]] +name = "argh_shared" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5693f39141bda5760ecc4111ab08da40565d1771038c4a0250f03457ec707531" +dependencies = [ + "serde", +] + [[package]] name = "argminmax" version = "0.6.1" @@ -4387,23 +4418,12 @@ name = "re_build_examples" version = "0.12.0-alpha.1+dev" dependencies = [ "anyhow", + "argh", "indicatif", "rayon", "re_build_tools", - "re_examples", "re_format", "serde", - "serde_yaml", -] - -[[package]] -name = "re_build_examples_manifest" -version = "0.12.0-alpha.1+dev" -dependencies = [ - "anyhow", - "re_build_tools", - "re_examples", - "serde", "serde_json", "serde_yaml", ] @@ -4531,16 +4551,6 @@ dependencies = [ "anyhow", ] -[[package]] -name = "re_examples" -version = "0.12.0-alpha.1+dev" -dependencies = [ - "anyhow", - "re_build_tools", - "serde", - "serde_yaml", -] - [[package]] name = "re_format" version = "0.12.0-alpha.1+dev" diff --git a/Cargo.toml b/Cargo.toml index 7964acbc3c00..441686f16155 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,6 @@ version = "0.12.0-alpha.1+dev" re_analytics = { path = "crates/re_analytics", version = "=0.12.0-alpha.1", default-features = false } re_arrow_store = { path = "crates/re_arrow_store", version = "=0.12.0-alpha.1", default-features = false } re_build_examples = { path = "crates/re_build_examples", version = "=0.12.0-alpha.1", default-features = false } -re_build_examples_manifest = { path = "crates/re_build_examples_manifest", version = "=0.12.0-alpha.1", default-features = false } re_build_info = { path = "crates/re_build_info", version = "=0.12.0-alpha.1", default-features = false } re_build_tools = { path = "crates/re_build_tools", version = "=0.12.0-alpha.1", default-features = false } re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "=0.12.0-alpha.1", default-features = false } @@ -38,7 +37,6 @@ re_data_source = { path = "crates/re_data_source", version = "=0.12.0-alpha.1", re_data_store = { path = "crates/re_data_store", version = "=0.12.0-alpha.1", default-features = false } re_data_ui = { path = "crates/re_data_ui", version = "=0.12.0-alpha.1", default-features = false } re_error = { path = "crates/re_error", version = "=0.12.0-alpha.1", default-features = false } -re_examples = { path = "crates/re_examples", version = "=0.12.0-alpha.1", default-features = false } re_format = { path = "crates/re_format", version = "=0.12.0-alpha.1", default-features = false } re_int_histogram = { path = "crates/re_int_histogram", version = "=0.12.0-alpha.1", default-features = false } re_log = { path = "crates/re_log", version = "=0.12.0-alpha.1", default-features = false } @@ -98,6 +96,7 @@ emath = "0.24.1" ahash = "0.8" anyhow = "1.0" arboard = { version = "3.2", default-features = false } +argh = "0.1.12" array-init = "2.1" arrow2 = "0.17" arrow2_convert = "0.5.0" diff --git a/crates/re_build_examples/Cargo.toml b/crates/re_build_examples/Cargo.toml index e3bc9937f1e6..369c7cb394a6 100644 --- a/crates/re_build_examples/Cargo.toml +++ b/crates/re_build_examples/Cargo.toml @@ -18,7 +18,6 @@ all-features = true [dependencies] re_build_tools.workspace = true -re_examples.workspace = true re_format.workspace = true # External @@ -27,3 +26,5 @@ indicatif.workspace = true rayon.workspace = true serde = { workspace = true, features = ["derive"] } serde_yaml.workspace = true +serde_json.workspace = true +argh.workspace = true diff --git a/crates/re_examples/src/lib.rs b/crates/re_build_examples/src/example.rs similarity index 80% rename from crates/re_examples/src/lib.rs rename to crates/re_build_examples/src/example.rs index 9bbb1a2b609d..450cff22f78d 100644 --- a/crates/re_examples/src/lib.rs +++ b/crates/re_build_examples/src/example.rs @@ -14,21 +14,10 @@ pub struct Example { pub thumbnail_dimensions: [u64; 2], pub script_path: PathBuf, pub script_args: Vec, - pub kind: ExampleKind, } -impl ExampleKind { - #[allow(clippy::match_like_matches_macro)] // harder to read - pub fn included_in(&self, channel: Channel) -> bool { - match (channel, self) { - (Channel::Nightly, ExampleKind::Nightly | ExampleKind::Demo) => true, - (Channel::Main, ExampleKind::Demo) => true, - _ => false, - } - } -} - -#[derive(Default, Clone, Copy)] +#[derive(Default, Clone, Copy, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] pub enum Channel { #[default] Main, @@ -65,8 +54,12 @@ impl Channel { continue; }; - let kind = ExampleKind::infer(readme.demo, readme.nightly); - if !kind.included_in(self) { + let Some(channel) = readme.channel else { + eprintln!("{name:?}: skipped"); + continue; + }; + + if channel != self { eprintln!("{name:?}: skipped"); continue; } @@ -81,7 +74,6 @@ impl Channel { thumbnail_dimensions: readme.thumbnail_dimensions, script_path: folder.path().join("main.py"), script_args: readme.build_args, - kind, }); } } @@ -106,33 +98,25 @@ impl Display for Channel { } impl FromStr for Channel { - type Err = (); + type Err = InvalidChannelName; fn from_str(s: &str) -> Result { match s { "main" => Ok(Self::Main), "nightly" => Ok(Self::Nightly), - _ => Err(()), + _ => Err(InvalidChannelName), } } } -#[derive(Clone, Copy)] -pub enum ExampleKind { - Ignore, - Demo, - Nightly, -} +#[derive(Debug)] +pub struct InvalidChannelName; -impl ExampleKind { - fn infer(demo: bool, nightly: bool) -> Self { - if demo && nightly { - Self::Nightly - } else if demo { - Self::Demo - } else { - Self::Ignore - } +impl std::error::Error for InvalidChannelName {} + +impl Display for InvalidChannelName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("invalid channel name") } } @@ -149,9 +133,7 @@ struct Frontmatter { #[serde(default)] thumbnail_dimensions: [u64; 2], #[serde(default)] - demo: bool, - #[serde(default)] - nightly: bool, + channel: Option, #[serde(default)] build_args: Vec, } diff --git a/crates/re_build_examples/src/main.rs b/crates/re_build_examples/src/main.rs index 74cc8db60a8b..0931dc73e203 100644 --- a/crates/re_build_examples/src/main.rs +++ b/crates/re_build_examples/src/main.rs @@ -1,203 +1,42 @@ -//! This build script collects all examples which should be part of our example page -//! and runs them to produce `.rrd` files. +//! This build script collects all examples which should be part of our example page, +//! and either runs them to produce `.rrd` files, or builds a manifest file which +//! serves as an index for the files. +//! +//! It identifies runnable examples by checking if they have `channel` set in +//! their `README.md` frontmatter. The available values are: +//! - `main` for simple/fast examples built on each PR and the `main` branch +//! - `nightly` for heavier examples built once per day //! -//! It identifies runnable examples by checking if they have `demo: true` set in -//! their `README.md` frontmatter. //! An example may also specify args to be run with via the frontmatter //! `build_args` string array. -use std::fs::create_dir_all; -use std::io::stdout; -use std::io::IsTerminal; -use std::path::Path; -use std::path::PathBuf; -use std::process::exit; -use std::process::Command; -use std::process::Output; -use std::time::Duration; - -use indicatif::MultiProgress; -use indicatif::ProgressBar; -use rayon::prelude::IntoParallelIterator; -use rayon::prelude::ParallelIterator; -use re_examples::{Channel, Example}; +mod example; +mod manifest; +mod rrd; -const USAGE: &str = "\ -Usage: [options] [output_dir] - -Options: - -h, --help Print help - --channel Determines which examples to build: `nightly`, `main` -"; +use argh::FromArgs; +use example::{Channel, Example}; fn main() -> anyhow::Result<()> { re_build_tools::set_output_cargo_build_instructions(false); - let args = Args::from_env(); - - create_dir_all(&args.output_dir)?; - - let examples = args.channel.examples()?; - let progress = MultiProgress::new(); - let results: Vec> = examples - .into_par_iter() - .map(|example| example.build(&progress, &args.output_dir)) - .collect(); - - let mut failed = false; - for result in results { - match result { - Ok(rrd_path) => { - if let Ok(metadata) = std::fs::metadata(&rrd_path) { - println!( - "Output: {} ({})", - rrd_path.display(), - re_format::format_bytes(metadata.len() as _) - ); - } else { - eprintln!("Missing rrd at {}", rrd_path.display()); - failed = true; - } - } - Err(err) => { - eprintln!("{err}"); - failed = true; - } - } + let args: Args = argh::from_env(); + match args.cmd { + Cmd::Rrd(cmd) => cmd.run(), + Cmd::Manifest(cmd) => cmd.run(), } - if failed { - anyhow::bail!("Failed to run some examples"); - } - - Ok(()) } +/// Build examples and their manifest. +#[derive(FromArgs)] struct Args { - output_dir: PathBuf, - channel: Channel, -} - -impl Args { - fn from_env() -> Self { - let mut output_dir: Option = None; - let mut channel: Option = None; - - let mut args = std::env::args().skip(1); - while let Some(arg) = args.next() { - match arg.as_str() { - "--help" | "-h" => { - println!("{USAGE}"); - exit(1); - } - "--channel" => { - let Some(value) = args.next() else { - eprintln!("Expected value after \"--channel\""); - println!("\n{USAGE}"); - std::process::exit(1); - }; - let Ok(value) = value.parse::() else { - eprintln!("Invalid \"--channel\", expected `nightly` or `main`"); - println!("\n{USAGE}"); - std::process::exit(1); - }; - channel = Some(value); - } - _ if arg.starts_with('-') => { - eprintln!("Unknown argument: {arg:?}"); - println!("\n{USAGE}"); - exit(1); - } - _ if output_dir.is_some() => { - eprintln!("Too many positional arguments"); - println!("\n{USAGE}"); - exit(1); - } - _ => output_dir = Some(PathBuf::from(arg)), - } - } - - let Some(output_dir) = output_dir else { - eprintln!("Missing argument \"output_dir\""); - exit(1); - }; - - let channel = channel.unwrap_or_default(); - - Args { - output_dir, - channel, - } - } + #[argh(subcommand)] + cmd: Cmd, } -trait Build { - /// Returns the path to the resulting `.rrd` file. - fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result; -} - -impl Build for Example { - fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result { - let rrd_path = output_dir.join(&self.name).with_extension("rrd"); - - let mut cmd = Command::new("python3"); - cmd.arg(self.script_path); - cmd.arg("--save").arg(&rrd_path); - cmd.args(self.script_args); - - let final_args = cmd - .get_args() - .map(|arg| arg.to_string_lossy().to_string()) - .collect::>(); - - // Configure flushing so that: - // * the resulting file size is deterministic - // * the file is chunked into small batches for better streaming - cmd.env("RERUN_FLUSH_TICK_SECS", 1_000_000_000.to_string()); - cmd.env("RERUN_FLUSH_NUM_BYTES", (128 * 1024).to_string()); - - let output = wait_for_output(cmd, &self.name, progress)?; - - if output.status.success() { - Ok(rrd_path) - } else { - anyhow::bail!( - "Failed to run `python3 {}`: \ - \nstdout: \ - \n{} \ - \nstderr: \ - \n{}", - final_args.join(" "), - String::from_utf8(output.stdout)?, - String::from_utf8(output.stderr)?, - ); - } - } -} - -fn wait_for_output( - mut cmd: Command, - name: &str, - progress: &MultiProgress, -) -> anyhow::Result { - let progress = progress.add(ProgressBar::new_spinner().with_message(name.to_owned())); - progress.enable_steady_tick(Duration::from_millis(100)); - - let output = cmd.output()?; - - let elapsed = progress.elapsed().as_secs_f64(); - let tick = if output.status.success() { - "✔" - } else { - "✘" - }; - let message = format!("{tick} {name} ({elapsed:.3}s)"); - - if stdout().is_terminal() { - progress.set_message(message); - progress.finish(); - } else { - println!("{message}"); - } - - Ok(output) +#[derive(FromArgs)] +#[argh(subcommand)] +enum Cmd { + Rrd(rrd::Rrd), + Manifest(manifest::Manifest), } diff --git a/crates/re_build_examples/src/manifest.rs b/crates/re_build_examples/src/manifest.rs new file mode 100644 index 000000000000..b1b5f11cf7d6 --- /dev/null +++ b/crates/re_build_examples/src/manifest.rs @@ -0,0 +1,149 @@ +use re_build_tools::Environment; + +use crate::{Channel, Example}; +use std::path::PathBuf; + +/// Collect examples in the repository and produce a manifest file. +/// +/// The manifest file contains example metadata, such as their names +/// and links to `.rrd` files. +#[derive(argh::FromArgs)] +#[argh(subcommand, name = "manifest")] +pub struct Manifest { + #[argh(positional, description = "output path for the manifest file")] + output_path: PathBuf, + + #[argh( + option, + description = "where examples are uploaded, e.g. `https://app.rerun.io/version/main`" + )] + base_url: Option, + + #[argh(option, description = "include only examples in this channel")] + channel: Channel, +} + +impl Manifest { + pub fn run(self) -> anyhow::Result<()> { + let build_env = Environment::detect(); + + let base_url = if matches!(self.channel, Channel::Nightly) { + "https://app.rerun.io/version/nightly".to_owned() + } else { + match &self.base_url { + Some(base_url) => base_url.clone(), + None => get_base_url(build_env)?, + } + }; + + let manifest = self + .channel + .examples()? + .into_iter() + .map(|example| ManifestEntry::new(example, &base_url)) + .collect::>(); + + if manifest.is_empty() { + anyhow::bail!("No examples found!"); + } + + std::fs::write(self.output_path, serde_json::to_string_pretty(&manifest)?)?; + + Ok(()) + } +} + +#[derive(serde::Serialize)] +struct ManifestEntry { + name: String, + title: String, + description: String, + tags: Vec, + rrd_url: String, + thumbnail: Thumbnail, +} + +impl ManifestEntry { + fn new(example: Example, base_url: &str) -> Self { + let name = example.name; + Self { + title: example.title, + description: example.description, + tags: example.tags, + rrd_url: format!("{base_url}/examples/{name}.rrd"), + thumbnail: Thumbnail { + url: example.thumbnail_url, + width: example.thumbnail_dimensions[0], + height: example.thumbnail_dimensions[1], + }, + name, + } + } +} + +#[derive(serde::Serialize)] +struct Thumbnail { + url: String, + width: u64, + height: u64, +} + +fn get_base_url(build_env: Environment) -> anyhow::Result { + // In the CondaBuild environment we can't trust the git_branch name -- if it exists + // at all it's going to be the feedstock branch-name, not our Rerun branch. However + // conda should ONLY be building released versions, so we want to version the manifest. + let versioned_manifest = matches!(build_env, Environment::CondaBuild) || { + let branch = re_build_tools::git_branch()?; + if branch == "main" || !re_build_tools::is_on_ci() { + // on `main` and local builds, use `version/main` + // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` + // on every commit to the `main` branch + return Ok("https://app.rerun.io/version/main".into()); + } + parse_release_version(&branch).is_some() + }; + + if versioned_manifest { + let metadata = re_build_tools::cargo_metadata()?; + let workspace_root = metadata + .root_package() + .ok_or_else(|| anyhow::anyhow!("failed to find workspace root"))?; + + // on `release-x.y.z` builds, use `version/{crate_version}` + // this will point to data uploaded by `.github/workflows/reusable_build_and_publish_web.yml` + return Ok(format!( + "https://app.rerun.io/version/{}", + workspace_root.version + )); + } + + // any other branch that is not `main`, use `commit/{sha}` + // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` + let sha = re_build_tools::git_commit_short_hash()?; + Ok(format!("https://app.rerun.io/commit/{sha}")) +} + +fn parse_release_version(branch: &str) -> Option<&str> { + // release-\d+.\d+.\d+(-alpha.\d+)? + + let version = branch.strip_prefix("release-")?; + + let (major, rest) = version.split_once('.')?; + major.parse::().ok()?; + let (minor, rest) = rest.split_once('.')?; + minor.parse::().ok()?; + let (patch, meta) = rest + .split_once('-') + .map_or((rest, None), |(p, m)| (p, Some(m))); + patch.parse::().ok()?; + + if let Some(meta) = meta { + let (kind, n) = meta.split_once('.')?; + if kind != "alpha" && kind != "rc" { + return None; + } + n.parse::().ok()?; + } + + Some(version) +} diff --git a/crates/re_build_examples/src/rrd.rs b/crates/re_build_examples/src/rrd.rs new file mode 100644 index 000000000000..25a87dda5e7b --- /dev/null +++ b/crates/re_build_examples/src/rrd.rs @@ -0,0 +1,136 @@ +use crate::{Channel, Example}; +use indicatif::MultiProgress; +use indicatif::ProgressBar; +use rayon::prelude::IntoParallelIterator; +use rayon::prelude::ParallelIterator; +use std::fs::create_dir_all; +use std::io::stdout; +use std::io::IsTerminal; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; +use std::process::Output; +use std::time::Duration; + +/// Collect examples in the repository and run them to produce `.rrd` files. +#[derive(argh::FromArgs)] +#[argh(subcommand, name = "rrd")] +pub struct Rrd { + #[argh(positional, description = "directory to output `rrd` files into")] + output_dir: PathBuf, + + #[argh(option, description = "include only examples in this channel")] + channel: Channel, +} + +impl Rrd { + pub fn run(self) -> anyhow::Result<()> { + create_dir_all(&self.output_dir)?; + + let examples = self.channel.examples()?; + let progress = MultiProgress::new(); + let results: Vec> = examples + .into_par_iter() + .map(|example| example.build(&progress, &self.output_dir)) + .collect(); + + let mut failed = false; + for result in results { + match result { + Ok(rrd_path) => { + if let Ok(metadata) = std::fs::metadata(&rrd_path) { + println!( + "Output: {} ({})", + rrd_path.display(), + re_format::format_bytes(metadata.len() as _) + ); + } else { + eprintln!("Missing rrd at {}", rrd_path.display()); + failed = true; + } + } + Err(err) => { + eprintln!("{err}"); + failed = true; + } + } + } + if failed { + anyhow::bail!("Failed to run some examples"); + } + + Ok(()) + } +} + +trait Build { + /// Returns the path to the resulting `.rrd` file. + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result; +} + +impl Build for Example { + fn build(self, progress: &MultiProgress, output_dir: &Path) -> anyhow::Result { + let rrd_path = output_dir.join(&self.name).with_extension("rrd"); + + let mut cmd = Command::new("python3"); + cmd.arg(self.script_path); + cmd.arg("--save").arg(&rrd_path); + cmd.args(self.script_args); + + let final_args = cmd + .get_args() + .map(|arg| arg.to_string_lossy().to_string()) + .collect::>(); + + // Configure flushing so that: + // * the resulting file size is deterministic + // * the file is chunked into small batches for better streaming + cmd.env("RERUN_FLUSH_TICK_SECS", 1_000_000_000.to_string()); + cmd.env("RERUN_FLUSH_NUM_BYTES", (128 * 1024).to_string()); + + let output = wait_for_output(cmd, &self.name, progress)?; + + if output.status.success() { + Ok(rrd_path) + } else { + anyhow::bail!( + "Failed to run `python3 {}`: \ + \nstdout: \ + \n{} \ + \nstderr: \ + \n{}", + final_args.join(" "), + String::from_utf8(output.stdout)?, + String::from_utf8(output.stderr)?, + ); + } + } +} + +fn wait_for_output( + mut cmd: Command, + name: &str, + progress: &MultiProgress, +) -> anyhow::Result { + let progress = progress.add(ProgressBar::new_spinner().with_message(name.to_owned())); + progress.enable_steady_tick(Duration::from_millis(100)); + + let output = cmd.output()?; + + let elapsed = progress.elapsed().as_secs_f64(); + let tick = if output.status.success() { + "✔" + } else { + "✘" + }; + let message = format!("{tick} {name} ({elapsed:.3}s)"); + + if stdout().is_terminal() { + progress.set_message(message); + progress.finish(); + } else { + println!("{message}"); + } + + Ok(output) +} diff --git a/crates/re_build_examples_manifest/Cargo.toml b/crates/re_build_examples_manifest/Cargo.toml deleted file mode 100644 index d3970c366033..000000000000 --- a/crates/re_build_examples_manifest/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "re_build_examples_manifest" -authors.workspace = true -description = "Build the rerun examples manifest JSON file" -edition.workspace = true -homepage.workspace = true -include.workspace = true -license.workspace = true -publish = false -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[package.metadata.docs.rs] -all-features = true - - -[dependencies] -re_build_tools.workspace = true -re_examples.workspace = true - -# External -anyhow.workspace = true -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true -serde_yaml.workspace = true diff --git a/crates/re_build_examples_manifest/src/main.rs b/crates/re_build_examples_manifest/src/main.rs deleted file mode 100644 index ec4741c23367..000000000000 --- a/crates/re_build_examples_manifest/src/main.rs +++ /dev/null @@ -1,225 +0,0 @@ -//! This build script generates the `examples_manifest.json` file. -//! It looks at all examples in the workspace (under `examples/python`), -//! and only includes those with `demo` set to `true` in their `README.md` -//! frontmatter. -//! -//! The URLs embedded in the `example_manifest.json` file point to a specific version. -//! This version is resolved according to the current environment: -//! -//! If the `CI` env var is set + the branch name is not `main`, then: -//! - On any `release-x.y.z` branch, the version is `version/x.y.z` -//! - On any other branch, the version is `commit/$COMMIT_SHORT_HASH` -//! -//! Otherwise, the version is `version/main`. This means local builds, -//! and builds on `main` point to `version/main`. - -use std::path::PathBuf; - -use re_build_tools::Environment; -use re_examples::{Channel, Example}; - -const USAGE: &str = "\ -Usage: [options] [output_path] - -Options: - -h, --help Print help - --base-url Where all examples are uploaded, e.g. `https://app.rerun.io/version/main` - --channel Determines which examples to build: `nightly`, `main` -"; - -fn main() -> anyhow::Result<()> { - re_build_tools::set_output_cargo_build_instructions(false); - - let args = Args::from_env(); - - let manifest = build_examples_manifest(Environment::detect(), &args)?; - std::fs::write(args.output_path, manifest)?; - - Ok(()) -} - -struct Args { - output_path: PathBuf, - base_url: Option, - channel: Channel, -} - -impl Args { - fn from_env() -> Self { - let mut output_path: Option = None; - let mut base_url: Option = None; - let mut channel: Option = None; - - let mut args = std::env::args().skip(1); - while let Some(arg) = args.next() { - match arg.as_str() { - "--help" | "-h" => { - println!("{USAGE}"); - std::process::exit(1); - } - "--base-url" => { - let Some(value) = args.next() else { - eprintln!("Expected value after \"--base-url\""); - println!("\n{USAGE}"); - std::process::exit(1); - }; - base_url = Some(value); - } - "--channel" => { - let Some(value) = args.next() else { - eprintln!("Expected value after \"--channel\""); - println!("\n{USAGE}"); - std::process::exit(1); - }; - let Ok(value) = value.parse::() else { - eprintln!("Invalid \"--channel\", expected `nightly` or `main`"); - println!("\n{USAGE}"); - std::process::exit(1); - }; - channel = Some(value); - } - _ if arg.starts_with('-') => { - eprintln!("Unknown argument: {arg:?}"); - println!("\n{USAGE}"); - std::process::exit(1); - } - _ if output_path.is_some() => { - eprintln!("Too many positional arguments"); - println!("\n{USAGE}"); - std::process::exit(1); - } - _ => output_path = Some(PathBuf::from(arg)), - } - } - - let Some(output_path) = output_path else { - eprintln!("Missing argument \"output_path\""); - std::process::exit(1); - }; - - let channel = channel.unwrap_or_default(); - - Args { - output_path, - base_url, - channel, - } - } -} - -fn build_examples_manifest(build_env: Environment, args: &Args) -> anyhow::Result { - let base_url = if matches!(args.channel, Channel::Nightly) { - "https://app.rerun.io/version/nightly".to_owned() - } else { - match &args.base_url { - Some(base_url) => base_url.clone(), - None => get_base_url(build_env)?, - } - }; - - let mut manifest = vec![]; - for example in args.channel.examples()? { - manifest.push(ManifestEntry::new(example, &base_url)); - } - - if manifest.is_empty() { - anyhow::bail!("No examples found!"); - } - - Ok(serde_json::to_string_pretty(&manifest)?) -} - -#[derive(serde::Serialize)] -struct ManifestEntry { - name: String, - title: String, - description: String, - tags: Vec, - rrd_url: String, - thumbnail: Thumbnail, -} - -impl ManifestEntry { - fn new(example: Example, base_url: &str) -> Self { - let name = example.name; - Self { - title: example.title, - description: example.description, - tags: example.tags, - rrd_url: format!("{base_url}/examples/{name}.rrd"), - thumbnail: Thumbnail { - url: example.thumbnail_url, - width: example.thumbnail_dimensions[0], - height: example.thumbnail_dimensions[1], - }, - name, - } - } -} - -#[derive(serde::Serialize)] -struct Thumbnail { - url: String, - width: u64, - height: u64, -} - -fn get_base_url(build_env: Environment) -> anyhow::Result { - // In the CondaBuild environment we can't trust the git_branch name -- if it exists - // at all it's going to be the feedstock branch-name, not our Rerun branch. However - // conda should ONLY be building released versions, so we want to version the manifest. - let versioned_manifest = matches!(build_env, Environment::CondaBuild) || { - let branch = re_build_tools::git_branch()?; - if branch == "main" || !re_build_tools::is_on_ci() { - // on `main` and local builds, use `version/main` - // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` - // on every commit to the `main` branch - return Ok("https://app.rerun.io/version/main".into()); - } - parse_release_version(&branch).is_some() - }; - - if versioned_manifest { - let metadata = re_build_tools::cargo_metadata()?; - let workspace_root = metadata - .root_package() - .ok_or_else(|| anyhow::anyhow!("failed to find workspace root"))?; - - // on `release-x.y.z` builds, use `version/{crate_version}` - // this will point to data uploaded by `.github/workflows/reusable_build_and_publish_web.yml` - return Ok(format!( - "https://app.rerun.io/version/{}", - workspace_root.version - )); - } - - // any other branch that is not `main`, use `commit/{sha}` - // this will point to data uploaded by `.github/workflows/reusable_upload_examples.yml` - let sha = re_build_tools::git_commit_short_hash()?; - Ok(format!("https://app.rerun.io/commit/{sha}")) -} - -fn parse_release_version(branch: &str) -> Option<&str> { - // release-\d+.\d+.\d+(-alpha.\d+)? - - let version = branch.strip_prefix("release-")?; - - let (major, rest) = version.split_once('.')?; - major.parse::().ok()?; - let (minor, rest) = rest.split_once('.')?; - minor.parse::().ok()?; - let (patch, meta) = rest - .split_once('-') - .map_or((rest, None), |(p, m)| (p, Some(m))); - patch.parse::().ok()?; - - if let Some(meta) = meta { - let (kind, n) = meta.split_once('.')?; - if kind != "alpha" && kind != "rc" { - return None; - } - n.parse::().ok()?; - } - - Some(version) -} diff --git a/crates/re_examples/Cargo.toml b/crates/re_examples/Cargo.toml deleted file mode 100644 index 416e0eb4a9ed..000000000000 --- a/crates/re_examples/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "re_examples" -authors.workspace = true -description = "Utils for rerun example builds" -edition.workspace = true -homepage.workspace = true -include.workspace = true -license.workspace = true -publish = false -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[package.metadata.docs.rs] -all-features = true - -[dependencies] -re_build_tools.workspace = true - -# External -anyhow.workspace = true -serde = { workspace = true, features = ["derive"] } -serde_yaml.workspace = true From ca8ebe3b68033a8a82a86199740c49a4f89aaf69 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:55:17 +0100 Subject: [PATCH 33/45] update usage --- .github/workflows/reusable_build_examples.yml | 2 +- .github/workflows/reusable_build_web.yml | 2 +- .github/workflows/reusable_publish_web.yml | 4 ++-- pixi.toml | 1 - .../screenshot_compare/build_screenshot_compare.py | 14 ++++++++++++-- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable_build_examples.yml b/.github/workflows/reusable_build_examples.yml index 72dc05af9e36..ef5a9351cefb 100644 --- a/.github/workflows/reusable_build_examples.yml +++ b/.github/workflows/reusable_build_examples.yml @@ -89,7 +89,7 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples \ + pixi run build-examples rrd \ --channel ${{ inputs.CHANNEL }} \ example_data diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index 8484ce005e36..a7fde9458fc4 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -76,7 +76,7 @@ jobs: full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" sha="$(echo $full_commit | cut -c1-7)" - pixi run build-examples-manifest \ + pixi run build-examples manifest \ --base-url "https://app.rerun.io/commit/$sha" \ --channel "${{ inputs.CHANNEL }}" \ web_viewer/examples_manifest.json diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml index 1e7450285c60..8e8f0d633855 100644 --- a/.github/workflows/reusable_publish_web.yml +++ b/.github/workflows/reusable_publish_web.yml @@ -107,7 +107,7 @@ jobs: - name: Build examples shell: bash run: | - pixi run build-examples \ + pixi run build-examples rrd \ --channel "main" \ web_viewer/examples @@ -117,7 +117,7 @@ jobs: full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || github.sha }}" sha="$(echo $full_commit | cut -c1-7)" - pixi run build-examples-manifest \ + pixi run build-examples manifest \ --base-url "https://app.rerun.io/commit/$sha" \ --channel "main" \ web_viewer/examples_manifest.json diff --git a/pixi.toml b/pixi.toml index 2ba3364deda1..dcea8ee34379 100644 --- a/pixi.toml +++ b/pixi.toml @@ -28,7 +28,6 @@ version = "0.1.0" # codegen = "cargo run --package re_types_builder -- " build-examples = "cargo run -q --locked -p re_build_examples --" -build-examples-manifest = "cargo run -q --locked -p re_build_examples_manifest --" # Assorted linting tasks fast-lint = "python scripts/fast_lint.py" diff --git a/scripts/screenshot_compare/build_screenshot_compare.py b/scripts/screenshot_compare/build_screenshot_compare.py index e2f039718f93..c62bd3016c46 100755 --- a/scripts/screenshot_compare/build_screenshot_compare.py +++ b/scripts/screenshot_compare/build_screenshot_compare.py @@ -151,11 +151,21 @@ def collect_code_examples() -> Iterable[Example]: def build_demo_examples() -> None: - cmd = ["cargo", "run", "--locked", "-p", "re_build_examples", "--", "example_data"] + # fmt: off + cmd = [ + "cargo", "run", "--locked", + "-p", "re_build_examples", "--", + "rrd", "example_data", + ] run(cmd, cwd=RERUN_DIR) - cmd = ["cargo", "run", "--locked", "-p", "re_build_examples_manifest", "--", "example_data/examples_manifest.json"] + cmd = [ + "cargo", "run", "--locked", + "-p", "re_build_examples", "--", + "manifest", "example_data/examples_manifest.json", + ] run(cmd, cwd=RERUN_DIR) + # fmt: on def collect_demo_examples() -> Iterable[Example]: From afce3e043cb7d8510806501217f258f1d28880ef Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:55:26 +0100 Subject: [PATCH 34/45] update ARCHITECTURE.md --- ARCHITECTURE.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index efe010ac8df0..ca5321481a4b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -177,10 +177,8 @@ Update instructions: |----------------------------|---------------------------------------------------------------| | re_build_info | Information about the build. Use together with re_build_tools | | re_build_tools | build.rs helpers for generating build info | -| re_examples | Utils for rerun example builds | | re_types_builder | Generates code for Rerun's SDKs from flatbuffers definitions. | -| re_build_examples | Build rerun example RRD files | -| re_build_examples_manifest | Build the rerun examples manifest JSON file | +| re_build_examples | Build rerun example RRD files and manifest | | re_build_web_viewer | Build the rerun web-viewer Wasm from source | From cc072cd59b3d477843470e1f36758287ecf30145 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:56:17 +0100 Subject: [PATCH 35/45] update example frontmatter --- examples/cpp/dna/README.md | 2 +- examples/python/arkit_scenes/README.md | 2 +- examples/python/depth_guided_stable_diffusion/README.md | 3 +-- examples/python/detect_and_track_objects/README.md | 3 +-- examples/python/dicom_mri/README.md | 2 +- examples/python/dna/README.md | 2 +- examples/python/human_pose_tracking/README.md | 2 +- examples/python/nuscenes/README.md | 3 +-- examples/python/objectron/README.md | 3 +-- examples/python/open_photogrammetry_format/README.md | 3 +-- examples/python/plots/README.md | 2 +- examples/python/raw_mesh/README.md | 3 +-- examples/python/rgbd/README.md | 3 +-- examples/python/segment_anything_model/README.md | 3 +-- examples/python/structure_from_motion/README.md | 2 +- examples/rust/dna/README.md | 2 +- 16 files changed, 16 insertions(+), 24 deletions(-) diff --git a/examples/cpp/dna/README.md b/examples/cpp/dna/README.md index a8a4178932c4..ba50cdc03d01 100644 --- a/examples/cpp/dna/README.md +++ b/examples/cpp/dna/README.md @@ -7,7 +7,7 @@ tags: [3d, api-example] description: "Simple example of logging point and line primitives to draw a 3D helix." thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/examples/python/arkit_scenes/README.md b/examples/python/arkit_scenes/README.md index e2da39fe359e..f45eb2b7e758 100644 --- a/examples/python/arkit_scenes/README.md +++ b/examples/python/arkit_scenes/README.md @@ -5,7 +5,7 @@ tags: [2D, 3D, depth, mesh, object-detection, pinhole-camera] description: "Visualize the ARKitScenes dataset, which contains color+depth images, the reconstructed mesh and labeled bounding boxes." thumbnail: https://static.rerun.io/arkit_scenes/fb9ec9e8d965369d39d51b17fc7fc5bae6be10cc/480w.png thumbnail_dimensions: [480, 243] -demo: true +channel: main --- diff --git a/examples/python/depth_guided_stable_diffusion/README.md b/examples/python/depth_guided_stable_diffusion/README.md index b3d802fdef11..94de9d5f6d83 100644 --- a/examples/python/depth_guided_stable_diffusion/README.md +++ b/examples/python/depth_guided_stable_diffusion/README.md @@ -4,8 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/depth_guid tags: [2D, depth, huggingface, stable-diffusion, tensor, text] thumbnail: https://static.rerun.io/depth_guided_stable_diffusion/a85516aba09f72649517891d767e15383ce7f4ea/480w.png thumbnail_dimensions: [480, 253] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/detect_and_track_objects/README.md b/examples/python/detect_and_track_objects/README.md index 506f0feedaa1..61ce69ee6f1b 100644 --- a/examples/python/detect_and_track_objects/README.md +++ b/examples/python/detect_and_track_objects/README.md @@ -5,8 +5,7 @@ tags: [2D, huggingface, object-detection, object-tracking, opencv] description: "Visualize object detection and segmentation using the Huggingface `transformers` library." thumbnail: https://static.rerun.io/detect_and_track_objects/59f5b97a8724f9037353409ab3d0b7cb47d1544b/480w.png thumbnail_dimensions: [480, 279] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/dicom_mri/README.md b/examples/python/dicom_mri/README.md index 367fbef7f05a..abdcb0bb9b96 100644 --- a/examples/python/dicom_mri/README.md +++ b/examples/python/dicom_mri/README.md @@ -5,7 +5,7 @@ tags: [tensor, mri, dicom] description: "Example using a DICOM MRI scan. This demonstrates the flexible tensor slicing capabilities of the Rerun viewer." thumbnail: https://static.rerun.io/dicom_mri/e39f34a1b1ddd101545007f43a61783e1d2e5f8e/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/examples/python/dna/README.md b/examples/python/dna/README.md index 05894d5e2295..5bea9a1a365e 100644 --- a/examples/python/dna/README.md +++ b/examples/python/dna/README.md @@ -7,7 +7,7 @@ tags: [3d, api-example] description: "Simple example of logging point and line primitives to draw a 3D helix." thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- diff --git a/examples/python/human_pose_tracking/README.md b/examples/python/human_pose_tracking/README.md index 4241e3b5defc..10d1eb168ac9 100644 --- a/examples/python/human_pose_tracking/README.md +++ b/examples/python/human_pose_tracking/README.md @@ -5,7 +5,7 @@ tags: [mediapipe, keypoint-detection, 2D, 3D] description: "Use the MediaPipe Pose solution to detect and track a human pose in video." thumbnail: https://static.rerun.io/human_pose_tracking/37d47fe7e3476513f9f58c38da515e2cd4a093f9/480w.png thumbnail_dimensions: [480, 272] -demo: true +channel: main --- diff --git a/examples/python/nuscenes/README.md b/examples/python/nuscenes/README.md index 6139d108c330..72b8a4b8ba98 100644 --- a/examples/python/nuscenes/README.md +++ b/examples/python/nuscenes/README.md @@ -5,8 +5,7 @@ tags: [lidar, 3D, 2D, object-detection, pinhole-camera] description: "Visualize the nuScenes dataset including lidar, radar, images, and bounding boxes." thumbnail: https://static.rerun.io/nuscenes/64a50a9d67cbb69ae872551989ee807b195f6b5d/480w.png thumbnail_dimensions: [480, 282] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/objectron/README.md b/examples/python/objectron/README.md index 4fd0406cabf8..0464566890d0 100644 --- a/examples/python/objectron/README.md +++ b/examples/python/objectron/README.md @@ -5,8 +5,7 @@ rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/objectron/src/ tags: [2D, 3D, object-detection, pinhole-camera] thumbnail: https://static.rerun.io/objectron/8ea3a37e6b4af2e06f8e2ea5e70c1951af67fea8/480w.png thumbnail_dimensions: [480, 268] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/open_photogrammetry_format/README.md b/examples/python/open_photogrammetry_format/README.md index 3c93a17466ab..f9d9c16a6038 100644 --- a/examples/python/open_photogrammetry_format/README.md +++ b/examples/python/open_photogrammetry_format/README.md @@ -4,8 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/open_photo tags: [2d, 3d, camera, photogrammetry] thumbnail: https://static.rerun.io/open_photogrammetry_format/603d5605f9670889bc8bce3365f16b831fce1eb1/480w.png thumbnail_dimensions: [480, 310] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/plots/README.md b/examples/python/plots/README.md index c2582f149f70..d8855202bd35 100644 --- a/examples/python/plots/README.md +++ b/examples/python/plots/README.md @@ -5,7 +5,7 @@ tags: [2d, plots, api-example] description: "Demonstration of various plots and charts supported by Rerun." thumbnail: https://static.rerun.io/plots/c5b91cf0bf2eaf91c71d6cdcd4fe312d4aeac572/480w.png thumbnail_dimensions: [480, 271] -demo: true +channel: main --- diff --git a/examples/python/raw_mesh/README.md b/examples/python/raw_mesh/README.md index ee85016f5f67..0d905bdab2ec 100644 --- a/examples/python/raw_mesh/README.md +++ b/examples/python/raw_mesh/README.md @@ -5,8 +5,7 @@ rust: https://github.com/rerun-io/rerun/tree/latest/examples/rust/raw_mesh/src/m tags: [mesh] thumbnail: https://static.rerun.io/raw_mesh/64bec98280b07794f7c9617f30ba2c20278601c3/480w.png thumbnail_dimensions: [480, 271] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/rgbd/README.md b/examples/python/rgbd/README.md index 248813098d63..1bca9fe0cc38 100644 --- a/examples/python/rgbd/README.md +++ b/examples/python/rgbd/README.md @@ -4,8 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/rgbd/main. tags: [2D, 3D, depth, nyud, pinhole-camera] thumbnail: https://static.rerun.io/rgbd/4109d29ed52fa0a8f980fcdd0e9673360c76879f/480w.png thumbnail_dimensions: [480, 254] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/segment_anything_model/README.md b/examples/python/segment_anything_model/README.md index 88907500b19d..4a423519ff49 100644 --- a/examples/python/segment_anything_model/README.md +++ b/examples/python/segment_anything_model/README.md @@ -4,8 +4,7 @@ python: https://github.com/rerun-io/rerun/tree/latest/examples/python/segment_an tags: [2D, sam, segmentation] thumbnail: https://static.rerun.io/segment_anything_model/6aa2651907efbcf81be55b343caa76b9de5f2138/480w.png thumbnail_dimensions: [480, 283] -demo: true -nightly: true +channel: nightly --- diff --git a/examples/python/structure_from_motion/README.md b/examples/python/structure_from_motion/README.md index 2e53a143dda9..6aa2079b3993 100644 --- a/examples/python/structure_from_motion/README.md +++ b/examples/python/structure_from_motion/README.md @@ -5,7 +5,7 @@ tags: [2D, 3D, colmap, pinhole-camera, time-series] description: "Visualize a sparse reconstruction by COLMAP, a general-purpose Structure-from-Motion and Multi-View Stereo pipeline." thumbnail: https://static.rerun.io/structure_from_motion/b17f8824291fa1102a4dc2184d13c91f92d2279c/480w.png thumbnail_dimensions: [480, 275] -demo: true +channel: main build_args: ["--dataset=colmap_fiat", "--resize=800x600"] --- diff --git a/examples/rust/dna/README.md b/examples/rust/dna/README.md index e6ef49761d52..84810c0c2fc0 100644 --- a/examples/rust/dna/README.md +++ b/examples/rust/dna/README.md @@ -7,7 +7,7 @@ tags: [3d, api-example] description: "Simple example of logging point and line primitives to draw a 3D helix." thumbnail: https://static.rerun.io/helix/f4c375546fa9d24f7cd3a1a715ebf75b2978817a/480w.png thumbnail_dimensions: [480, 285] -demo: true +channel: main --- From 9b4e334955bd2a1e53d74240490a746b1b3036ae Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:05:08 +0100 Subject: [PATCH 36/45] fix versions --- examples/python/arkit_scenes/requirements.txt | 2 +- examples/python/human_pose_tracking/requirements.txt | 2 +- examples/python/lidar/requirements.txt | 2 +- examples/python/nuscenes/requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/python/arkit_scenes/requirements.txt b/examples/python/arkit_scenes/requirements.txt index 723970a7529d..76da574205e0 100644 --- a/examples/python/arkit_scenes/requirements.txt +++ b/examples/python/arkit_scenes/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib>=3.8.2 numpy opencv-python pandas diff --git a/examples/python/human_pose_tracking/requirements.txt b/examples/python/human_pose_tracking/requirements.txt index 574bdb7345ef..b83d297cf823 100644 --- a/examples/python/human_pose_tracking/requirements.txt +++ b/examples/python/human_pose_tracking/requirements.txt @@ -1,4 +1,4 @@ -mediapipe>=0.10.1 +mediapipe>=0.10.9 numpy opencv-python<4.6 # Avoid opencv-4.6 since it rotates images incorrectly (https://github.com/opencv/opencv/issues/22088) requests>=2.31,<3 diff --git a/examples/python/lidar/requirements.txt b/examples/python/lidar/requirements.txt index 78d507945c75..2a4536107e25 100644 --- a/examples/python/lidar/requirements.txt +++ b/examples/python/lidar/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib>=3.8.2 numpy nuscenes-devkit requests diff --git a/examples/python/nuscenes/requirements.txt b/examples/python/nuscenes/requirements.txt index 78d507945c75..2a4536107e25 100644 --- a/examples/python/nuscenes/requirements.txt +++ b/examples/python/nuscenes/requirements.txt @@ -1,4 +1,4 @@ -matplotlib +matplotlib>=3.8.2 numpy nuscenes-devkit requests From 6a461b54906ee8357ea838b811c34c9ab3990ca6 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:07:18 +0100 Subject: [PATCH 37/45] document frontmatter `channel` --- examples/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/README.md b/examples/README.md index ea04a9f1b13f..46d8dc60e6fa 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,3 +47,8 @@ The `manifest.yml` file describes the structure of the examples contained in thi You can base your example off of `python/template` or `rust/template`. Once it's ready to be displayed in the docs, add it to the [manifest](./manifest.yml). + +If you want to run the example on CI and include it in the in-viewer example page, +add a `channel` entry to its README frontmatter. The available channels right now are: +- `main` for simple/fast examples built on each PR and the `main` branch +- `nightly` for heavier examples built once per day From 4aac2b13f90446244e183937ff26dad9828ee358 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:08:29 +0100 Subject: [PATCH 38/45] spacing --- crates/re_build_examples/src/example.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/re_build_examples/src/example.rs b/crates/re_build_examples/src/example.rs index 450cff22f78d..7cdcbdfc7bb3 100644 --- a/crates/re_build_examples/src/example.rs +++ b/crates/re_build_examples/src/example.rs @@ -124,16 +124,22 @@ impl Display for InvalidChannelName { struct Frontmatter { #[serde(default)] title: String, + #[serde(default)] tags: Vec, + #[serde(default)] description: String, + #[serde(default)] thumbnail: String, + #[serde(default)] thumbnail_dimensions: [u64; 2], + #[serde(default)] channel: Option, + #[serde(default)] build_args: Vec, } From d210ec1d6e22e435bcdb2c3e1ecd46ebb5df2f87 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:08:37 +0100 Subject: [PATCH 39/45] rename error variable --- crates/re_build_examples/src/example.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/re_build_examples/src/example.rs b/crates/re_build_examples/src/example.rs index 7cdcbdfc7bb3..b6deb75f8eeb 100644 --- a/crates/re_build_examples/src/example.rs +++ b/crates/re_build_examples/src/example.rs @@ -156,9 +156,9 @@ fn parse_frontmatter>(path: P) -> anyhow::Result Date: Thu, 14 Dec 2023 13:09:26 +0100 Subject: [PATCH 40/45] update error message --- crates/re_build_examples/src/example.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_build_examples/src/example.rs b/crates/re_build_examples/src/example.rs index b6deb75f8eeb..b27ad010fa06 100644 --- a/crates/re_build_examples/src/example.rs +++ b/crates/re_build_examples/src/example.rs @@ -153,7 +153,7 @@ fn parse_frontmatter>(path: P) -> anyhow::Result Date: Thu, 14 Dec 2023 14:12:52 +0100 Subject: [PATCH 41/45] only set default manifest url on nightly --- .github/workflows/reusable_build_web.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml index a7fde9458fc4..518a967cef9a 100644 --- a/.github/workflows/reusable_build_web.yml +++ b/.github/workflows/reusable_build_web.yml @@ -10,10 +10,6 @@ on: required: false type: string default: "prerelease" - EXTRA_FLAGS: - required: false - type: string - default: "" CHANNEL: # `nightly` or `main` required: true type: string @@ -63,10 +59,11 @@ jobs: - name: Build web-viewer (release) shell: bash - env: - DEFAULT_EXAMPLES_MANIFEST_URL: "https://app.rerun.io/version/nightly/examples_manifest.json" run: | - cargo run --locked -p re_build_web_viewer -- --release ${{ inputs.EXTRA_FLAGS }} + if [ ${{ inputs.CHANNEL }} = "nightly" ]; then + export DEFAULT_EXAMPLES_MANIFEST_URL="https://app.rerun.io/version/nightly/examples_manifest.json" + fi + cargo run --locked -p re_build_web_viewer -- --release # We build a single manifest pointing to the `commit` # All the `pr`, `main`, release tag, etc. variants will always just point to the resolved commit From 2d415d6dfd840a75d77083a886304da87b80a73d Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 14:13:25 +0100 Subject: [PATCH 42/45] temp --- .github/workflows/nightly.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 51e4ad64d56f..6fe1199320bf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,10 +2,14 @@ name: Nightly on: workflow_dispatch: - schedule: - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule - # 12:15 UTC, every day - - cron: "15 12 * * *" + pull_request: + types: + - opened + - synchronize + # schedule: + # # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule + # # 12:15 UTC, every day + # - cron: "15 12 * * *" jobs: build-web: From 1b1bd70dc89f784fffd59059264b5f83341e02ce Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 14:26:54 +0100 Subject: [PATCH 43/45] fix track_sizes not depending on web build --- .github/workflows/on_pull_request.yml | 2 +- .github/workflows/on_push_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index 3ca8d0e5703c..d5df679648c7 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -138,7 +138,7 @@ jobs: track-sizes: name: "Track Sizes" if: github.event.pull_request.head.repo.owner.login == 'rerun-io' - needs: [build-examples] + needs: [build-web, build-examples] uses: ./.github/workflows/reusable_track_size.yml with: CONCURRENCY: push-${{ github.ref_name }} diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml index 61bd55a3e261..4f623d077c65 100644 --- a/.github/workflows/on_push_main.yml +++ b/.github/workflows/on_push_main.yml @@ -88,7 +88,7 @@ jobs: track-sizes: name: "Track Sizes" - needs: [build-examples] + needs: [build-web, build-examples] uses: ./.github/workflows/reusable_track_size.yml with: CONCURRENCY: push-${{ github.ref_name }} From 319489c123a546b46f64295166dd1bf5239b561e Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:33:51 +0100 Subject: [PATCH 44/45] back to schedule only --- .github/workflows/nightly.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6fe1199320bf..51e4ad64d56f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2,14 +2,10 @@ name: Nightly on: workflow_dispatch: - pull_request: - types: - - opened - - synchronize - # schedule: - # # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule - # # 12:15 UTC, every day - # - cron: "15 12 * * *" + schedule: + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule + # 12:15 UTC, every day + - cron: "15 12 * * *" jobs: build-web: From 60e864c3de5ac0d1ef5bc09b3de85f0ea94d4a04 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:20:34 +0100 Subject: [PATCH 45/45] sort --- crates/re_build_examples/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_build_examples/Cargo.toml b/crates/re_build_examples/Cargo.toml index 369c7cb394a6..71c1a63cf7ff 100644 --- a/crates/re_build_examples/Cargo.toml +++ b/crates/re_build_examples/Cargo.toml @@ -22,9 +22,9 @@ re_format.workspace = true # External anyhow.workspace = true +argh.workspace = true indicatif.workspace = true rayon.workspace = true serde = { workspace = true, features = ["derive"] } serde_yaml.workspace = true serde_json.workspace = true -argh.workspace = true