Better query plan viewing experience

Cargo.toml

@@ -54,6 +54,7 @@ members = [
   "crates/api_common",
   "crates/apub",
   "crates/utils",
+  "crates/db_perf",
   "crates/db_schema",
   "crates/db_views",
   "crates/db_views_actor",
@@ -156,6 +157,7 @@ tokio-postgres = "0.7.10"
 tokio-postgres-rustls = "0.10.0"
 enum-map = "2.7"
 moka = { version = "0.12.1", features = ["future"] }
+clap = { version = "4.4.11", features = ["derive"] }
 
 [dependencies]
 lemmy_api = { workspace = true }
@@ -192,5 +194,5 @@ futures-util = { workspace = true }
 chrono = { workspace = true }
 prometheus = { version = "0.13.3", features = ["process"] }
 serial_test = { workspace = true }
-clap = { version = "4.4.11", features = ["derive"] }
+clap = { workspace = true }
 actix-web-prom = "0.7.0"

crates/db_perf/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "lemmy_db_perf"
+version.workspace = true
+edition.workspace = true
+description.workspace = true
+license.workspace = true
+homepage.workspace = true
+documentation.workspace = true
+repository.workspace = true
+
+
+[lints]
+workspace = true
+
+[dependencies]
+clap = { workspace = true }
+diesel = { workspace = true }
+diesel-async = { workspace = true }
+lemmy_db_schema = { workspace = true }
+lemmy_db_views = { workspace = true, features = ["full"] }
+lemmy_utils = { workspace = true }
+tokio = { workspace = true }

crates/db_perf/src/main.rs

@@ -0,0 +1,174 @@
+use clap::Parser;
+use diesel::{
+  dsl::{self, sql},
+  sql_types,
+  ExpressionMethods,
+  IntoSql,
+};
+use diesel_async::{RunQueryDsl, SimpleAsyncConnection};
+use lemmy_db_schema::{
+  schema::post,
+  source::{
+    community::{Community, CommunityInsertForm},
+    instance::Instance,
+    person::{Person, PersonInsertForm},
+  },
+  traits::Crud,
+  utils::{
+    build_db_pool,
+    get_conn,
+    now,
+    series::{self, ValuesFromSeries},
+  },
+  SortType,
+};
+use lemmy_db_views::{post_view::PostQuery, structs::PaginationCursor};
+use lemmy_utils::error::LemmyResult;
+use std::num::NonZeroU32;
+
+#[derive(Parser, Debug)]
+struct CmdArgs {
+  #[arg(long, default_value_t = 3.try_into().unwrap())]
+  communities: NonZeroU32,
+  #[arg(long, default_value_t = 3.try_into().unwrap())]
+  people: NonZeroU32,
+  #[arg(long, default_value_t = 100000.try_into().unwrap())]
+  posts: NonZeroU32,
+  #[arg(long, default_value_t = 0)]
+  read_post_pages: u32,
+  #[arg(long)]
+  explain_insertions: bool,
+}
+
+#[tokio::main]
+async fn main() -> LemmyResult<()> {
+  if let Err(err) = try_main().await {
+    println!("😂 Error: {err:?}");
+  }
+  if let Ok(path) = std::env::var("PGDATA") {
+    println!("🪵 query plans and error details written in {path}/log");
+  }
+
+  Ok(())
+}
+
+async fn try_main() -> LemmyResult<()> {
+  let args = CmdArgs::parse();
+  let pool = &build_db_pool().await?;
+  let pool = &mut pool.into();
+  let conn = &mut get_conn(pool).await?;
+
+  if args.explain_insertions {
+    // log_nested_statements is enabled to log trigger execution
+    conn
+      .batch_execute(
+        "SET auto_explain.log_min_duration = 0; SET auto_explain.log_nested_statements = on;",
+      )
+      .await?;
+  }
+
+  let instance = Instance::read_or_create(&mut conn.into(), "reddit.com".to_owned()).await?;
+
+  println!("🫃 creating {} people", args.people);
+  let mut person_ids = vec![];
+  for i in 0..args.people.get() {
+    let form = PersonInsertForm::builder()
+      .name(format!("p{i}"))
+      .public_key("pubkey".to_owned())
+      .instance_id(instance.id)
+      .build();
+    person_ids.push(Person::create(&mut conn.into(), &form).await?.id);
+  }
+
+  println!("🌍 creating {} communities", args.communities);
+  let mut community_ids = vec![];
+  for i in 0..args.communities.get() {
+    let form = CommunityInsertForm::builder()
+      .name(format!("c{i}"))
+      .title(i.to_string())
+      .instance_id(instance.id)
+      .build();
+    community_ids.push(Community::create(&mut conn.into(), &form).await?.id);
+  }
+
+  let post_batches = args.people.get() * args.communities.get();
+  let posts_per_batch = args.posts.get() / post_batches;
+  let num_posts = post_batches * posts_per_batch;
+  println!(
+    "📜 creating {} posts ({} featured in community)",
+    num_posts, post_batches
+  );
+  let mut num_inserted_posts = 0;
+  // TODO: progress bar
+  for person_id in &person_ids {
+    for community_id in &community_ids {
+      let n = dsl::insert_into(post::table)
+        .values(ValuesFromSeries {
+          start: 1,
+          stop: posts_per_batch.into(),
+          selection: (
+            "AAAAAAAAAAA".into_sql::<sql_types::Text>(),
+            person_id.into_sql::<sql_types::Integer>(),
+            community_id.into_sql::<sql_types::Integer>(),
+            series::current_value.eq(1),
+            now()
+              - sql::<sql_types::Interval>("make_interval(secs => ")
+                .bind::<sql_types::BigInt, _>(series::current_value)
+                .sql(")"),
+          ),
+        })
+        .into_columns((
+          post::name,
+          post::creator_id,
+          post::community_id,
+          post::featured_community,
+          post::published,
+        ))
+        .execute(conn)
+        .await?;
+      num_inserted_posts += n;
+    }
+  }
+  // Make sure the println above shows the correct amount
+  assert_eq!(num_inserted_posts, num_posts as usize);
+
+  // Enable auto_explain
+  conn
+    .batch_execute(
+      "SET auto_explain.log_min_duration = 0; SET auto_explain.log_nested_statements = off;",
+    )
+    .await?;
+
+  // TODO: show execution duration stats
+  let mut page_after = None;
+  for page_num in 1..=args.read_post_pages {
+    println!(
+      "👀 getting page {page_num} of posts (pagination cursor used: {})",
+      page_after.is_some()
+    );
+
+    // TODO: include local_user
+    let post_views = PostQuery {
+      community_id: community_ids.get(0).cloned(),
+      sort: Some(SortType::New),
+      limit: Some(20),
+      page_after,
+      ..Default::default()
+    }
+    .list(&mut conn.into())
+    .await?;
+
+    if let Some(post_view) = post_views.into_iter().last() {
+      println!("👀 getting pagination cursor data for next page");
+      let cursor_data = PaginationCursor::after_post(&post_view)
+        .read(&mut conn.into())
+        .await?;
+      page_after = Some(cursor_data);
+    } else {
+      println!("👀 reached empty page");
+      break;
+    }
+  }
+
+  Ok(())
+}

crates/db_schema/Cargo.toml

@@ -76,6 +76,7 @@ tokio-postgres = { workspace = true, optional = true }
 tokio-postgres-rustls = { workspace = true, optional = true }
 rustls = { workspace = true, optional = true }
 uuid = { workspace = true, features = ["v4"] }
+anyhow = { workspace = true }
 
 [dev-dependencies]
 serial_test = { workspace = true }

crates/db_schema/src/impls/actor_language.rs

@@ -96,16 +96,18 @@ impl LocalUserLanguage {
             .execute(conn)
             .await?;
 
-          for l in lang_ids {
-            let form = LocalUserLanguageForm {
+          let forms = lang_ids
+            .into_iter()
+            .map(|l| LocalUserLanguageForm {
               local_user_id: for_local_user_id,
               language_id: l,
-            };
-            insert_into(local_user_language)
-              .values(form)
-              .get_result::<Self>(conn)
-              .await?;
-          }
+            })
+            .collect::<Vec<_>>();
+
+          insert_into(local_user_language)
+            .values(forms)
+            .execute(conn)
+            .await?;
           Ok(())
         }) as _
       })
@@ -164,16 +166,18 @@ impl SiteLanguage {
             .execute(conn)
             .await?;
 
-          for l in lang_ids {
-            let form = SiteLanguageForm {
+          let forms = lang_ids
+            .into_iter()
+            .map(|l| SiteLanguageForm {
               site_id: for_site_id,
               language_id: l,
-            };
-            insert_into(site_language)
-              .values(form)
-              .get_result::<Self>(conn)
-              .await?;
-          }
+            })
+            .collect::<Vec<_>>();
+
+          insert_into(site_language)
+            .values(forms)
+            .get_result::<Self>(conn)
+            .await?;
 
           CommunityLanguage::limit_languages(conn, instance_id).await?;