From 1d1136eaad120279867c10fa4edb5a1328a51de3 Mon Sep 17 00:00:00 2001 From: eth3lbert Date: Tue, 5 Nov 2024 07:14:51 +0800 Subject: [PATCH 1/5] controllers/summary: Extract `num_crates` and `num_downloads` --- src/controllers/summary.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/controllers/summary.rs b/src/controllers/summary.rs index 0881f37a7f8..094510c5b0f 100644 --- a/src/controllers/summary.rs +++ b/src/controllers/summary.rs @@ -8,8 +8,9 @@ use crate::util::diesel::Conn; use crate::util::errors::AppResult; use crate::views::{EncodableCategory, EncodableCrate, EncodableKeyword}; use axum::Json; -use diesel::prelude::*; +use diesel::QueryResult; use diesel_async::async_connection_wrapper::AsyncConnectionWrapper; +use diesel_async::AsyncPgConnection; use serde_json::Value; /// Handles the `GET /summary` route. @@ -22,16 +23,27 @@ pub async fn summary(state: AppState) -> AppResult> { .map(Category::into) .collect::>(); + async fn inner(conn: &mut AsyncPgConnection) -> QueryResult<(i64, i64)> { + use diesel::QueryDsl; + use diesel_async::RunQueryDsl; + + let num_crates: i64 = crates::table.count().get_result(conn).await?; + let num_downloads: i64 = metadata::table + .select(metadata::total_downloads) + .get_result(conn) + .await?; + + Ok((num_crates, num_downloads)) + } + + let (num_crates, num_downloads) = inner(&mut conn).await?; + spawn_blocking(move || { + use diesel::prelude::*; let conn: &mut AsyncConnectionWrapper<_> = &mut conn.into(); let config = &state.config; - let num_crates: i64 = crates::table.count().get_result(conn)?; - let num_downloads: i64 = metadata::table - .select(metadata::total_downloads) - .get_result(conn)?; - fn encode_crates( conn: &mut impl Conn, data: Vec, From 1bbd510cc4cfa6e948cd397395957e72bdcfaf61 Mon Sep 17 00:00:00 2001 From: eth3lbert Date: Tue, 5 Nov 2024 07:22:02 +0800 Subject: [PATCH 2/5] controllers/summary: Extract `new_crates` and `just_updated` --- src/controllers/summary.rs | 62 +++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/controllers/summary.rs b/src/controllers/summary.rs index 094510c5b0f..a0a4e463a45 100644 --- a/src/controllers/summary.rs +++ b/src/controllers/summary.rs @@ -23,8 +23,12 @@ pub async fn summary(state: AppState) -> AppResult> { .map(Category::into) .collect::>(); - async fn inner(conn: &mut AsyncPgConnection) -> QueryResult<(i64, i64)> { - use diesel::QueryDsl; + async fn inner( + conn: &mut AsyncPgConnection, + ) -> QueryResult<(i64, i64, Vec, Vec)> { + use diesel::{ + ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper, + }; use diesel_async::RunQueryDsl; let num_crates: i64 = crates::table.count().get_result(conn).await?; @@ -33,10 +37,40 @@ pub async fn summary(state: AppState) -> AppResult> { .get_result(conn) .await?; - Ok((num_crates, num_downloads)) + let selection = ( + Crate::as_select(), + crate_downloads::downloads, + recent_crate_downloads::downloads.nullable(), + versions::num.nullable(), + versions::yanked.nullable(), + ); + + let new_crates = crates::table + .inner_join(crate_downloads::table) + .left_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .order(crates::created_at.desc()) + .select(selection) + .limit(10) + .load(conn) + .await?; + let just_updated = crates::table + .inner_join(crate_downloads::table) + .left_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .filter(crates::updated_at.ne(crates::created_at)) + .order(crates::updated_at.desc()) + .select(selection) + .limit(10) + .load(conn) + .await?; + + Ok((num_crates, num_downloads, new_crates, just_updated)) } - let (num_crates, num_downloads) = inner(&mut conn).await?; + let (num_crates, num_downloads, new_crates, just_updated) = inner(&mut conn).await?; spawn_blocking(move || { use diesel::prelude::*; @@ -79,26 +113,6 @@ pub async fn summary(state: AppState) -> AppResult> { versions::yanked.nullable(), ); - let new_crates = crates::table - .inner_join(crate_downloads::table) - .left_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .order(crates::created_at.desc()) - .select(selection) - .limit(10) - .load(conn)?; - let just_updated = crates::table - .inner_join(crate_downloads::table) - .left_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .filter(crates::updated_at.ne(crates::created_at)) - .order(crates::updated_at.desc()) - .select(selection) - .limit(10) - .load(conn)?; - let mut most_downloaded_query = crates::table .inner_join(crate_downloads::table) .left_join(recent_crate_downloads::table) From c5973bb293de42c1dcde0954d4b28417e990af7a Mon Sep 17 00:00:00 2001 From: eth3lbert Date: Tue, 5 Nov 2024 07:29:28 +0800 Subject: [PATCH 3/5] controllers/summary: Extract `most_downloaded` and `most_recently_downloaded` --- src/controllers/summary.rs | 98 +++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/src/controllers/summary.rs b/src/controllers/summary.rs index a0a4e463a45..67cb30f208b 100644 --- a/src/controllers/summary.rs +++ b/src/controllers/summary.rs @@ -25,7 +25,8 @@ pub async fn summary(state: AppState) -> AppResult> { async fn inner( conn: &mut AsyncPgConnection, - ) -> QueryResult<(i64, i64, Vec, Vec)> { + config: &crate::config::Server, + ) -> QueryResult<(i64, i64, Vec, Vec, Vec, Vec)> { use diesel::{ ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper, }; @@ -67,17 +68,64 @@ pub async fn summary(state: AppState) -> AppResult> { .load(conn) .await?; - Ok((num_crates, num_downloads, new_crates, just_updated)) + let mut most_downloaded_query = crates::table + .inner_join(crate_downloads::table) + .left_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .into_boxed(); + if !config.excluded_crate_names.is_empty() { + most_downloaded_query = + most_downloaded_query.filter(crates::name.ne_all(&config.excluded_crate_names)); + } + let most_downloaded = most_downloaded_query + .then_order_by(crate_downloads::downloads.desc()) + .select(selection) + .limit(10) + .load(conn) + .await?; + + let mut most_recently_downloaded_query = crates::table + .inner_join(crate_downloads::table) + .inner_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .into_boxed(); + if !config.excluded_crate_names.is_empty() { + most_recently_downloaded_query = most_recently_downloaded_query + .filter(crates::name.ne_all(&config.excluded_crate_names)); + } + let most_recently_downloaded = most_recently_downloaded_query + .then_order_by(recent_crate_downloads::downloads.desc()) + .select(selection) + .limit(10) + .load(conn) + .await?; + + Ok(( + num_crates, + num_downloads, + new_crates, + just_updated, + most_downloaded, + most_recently_downloaded, + )) } - let (num_crates, num_downloads, new_crates, just_updated) = inner(&mut conn).await?; + let config = &state.config; + let ( + num_crates, + num_downloads, + new_crates, + just_updated, + most_downloaded, + most_recently_downloaded, + ) = inner(&mut conn, config).await?; spawn_blocking(move || { use diesel::prelude::*; let conn: &mut AsyncConnectionWrapper<_> = &mut conn.into(); - let config = &state.config; - fn encode_crates( conn: &mut impl Conn, data: Vec, @@ -105,46 +153,6 @@ pub async fn summary(state: AppState) -> AppResult> { .collect() } - let selection = ( - Crate::as_select(), - crate_downloads::downloads, - recent_crate_downloads::downloads.nullable(), - versions::num.nullable(), - versions::yanked.nullable(), - ); - - let mut most_downloaded_query = crates::table - .inner_join(crate_downloads::table) - .left_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .into_boxed(); - if !config.excluded_crate_names.is_empty() { - most_downloaded_query = - most_downloaded_query.filter(crates::name.ne_all(&config.excluded_crate_names)); - } - let most_downloaded = most_downloaded_query - .then_order_by(crate_downloads::downloads.desc()) - .select(selection) - .limit(10) - .load(conn)?; - - let mut most_recently_downloaded_query = crates::table - .inner_join(crate_downloads::table) - .inner_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .into_boxed(); - if !config.excluded_crate_names.is_empty() { - most_recently_downloaded_query = most_recently_downloaded_query - .filter(crates::name.ne_all(&config.excluded_crate_names)); - } - let most_recently_downloaded = most_recently_downloaded_query - .then_order_by(recent_crate_downloads::downloads.desc()) - .select(selection) - .limit(10) - .load(conn)?; - let popular_keywords = keywords::table .order(keywords::crates_cnt.desc()) .limit(10) From 1c11a638223ed29d61d2de74d0f4c9e1551569e0 Mon Sep 17 00:00:00 2001 From: eth3lbert Date: Tue, 5 Nov 2024 07:39:09 +0800 Subject: [PATCH 4/5] controllers/summary: Extract `popular_keywords` and `encode_crates()` --- src/controllers/summary.rs | 109 +++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/src/controllers/summary.rs b/src/controllers/summary.rs index 67cb30f208b..3bcf455832e 100644 --- a/src/controllers/summary.rs +++ b/src/controllers/summary.rs @@ -3,13 +3,10 @@ use crate::models::{Category, Crate, CrateVersions, Keyword, TopVersions, Versio use crate::schema::{ crate_downloads, crates, default_versions, keywords, metadata, recent_crate_downloads, versions, }; -use crate::tasks::spawn_blocking; -use crate::util::diesel::Conn; use crate::util::errors::AppResult; use crate::views::{EncodableCategory, EncodableCrate, EncodableKeyword}; use axum::Json; use diesel::QueryResult; -use diesel_async::async_connection_wrapper::AsyncConnectionWrapper; use diesel_async::AsyncPgConnection; use serde_json::Value; @@ -26,7 +23,15 @@ pub async fn summary(state: AppState) -> AppResult> { async fn inner( conn: &mut AsyncPgConnection, config: &crate::config::Server, - ) -> QueryResult<(i64, i64, Vec, Vec, Vec, Vec)> { + ) -> QueryResult<( + i64, + i64, + Vec, + Vec, + Vec, + Vec, + Vec, + )> { use diesel::{ ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper, }; @@ -102,6 +107,15 @@ pub async fn summary(state: AppState) -> AppResult> { .load(conn) .await?; + let popular_keywords = keywords::table + .order(keywords::crates_cnt.desc()) + .limit(10) + .load(conn) + .await? + .into_iter() + .map(Keyword::into) + .collect::>(); + Ok(( num_crates, num_downloads, @@ -109,6 +123,7 @@ pub async fn summary(state: AppState) -> AppResult> { just_updated, most_downloaded, most_recently_downloaded, + popular_keywords, )) } @@ -120,59 +135,49 @@ pub async fn summary(state: AppState) -> AppResult> { just_updated, most_downloaded, most_recently_downloaded, + popular_keywords, ) = inner(&mut conn, config).await?; - spawn_blocking(move || { - use diesel::prelude::*; - let conn: &mut AsyncConnectionWrapper<_> = &mut conn.into(); - - fn encode_crates( - conn: &mut impl Conn, - data: Vec, - ) -> AppResult> { - let krates = data.iter().map(|(c, ..)| c).collect::>(); - let versions: Vec = krates.versions().load(conn)?; - versions - .grouped_by(&krates) - .into_iter() - .map(TopVersions::from_versions) - .zip(data) - .map( - |(top_versions, (krate, total, recent, default_version, yanked))| { - Ok(EncodableCrate::from_minimal( - krate, - default_version.as_deref(), - yanked, - Some(&top_versions), - false, - total, - recent, - )) - }, - ) - .collect() - } + async fn encode_crates( + conn: &mut AsyncPgConnection, + data: Vec, + ) -> AppResult> { + use diesel::GroupedBy; + use diesel_async::RunQueryDsl; - let popular_keywords = keywords::table - .order(keywords::crates_cnt.desc()) - .limit(10) - .load(conn)? + let krates = data.iter().map(|(c, ..)| c).collect::>(); + let versions: Vec = krates.versions().load(conn).await?; + versions + .grouped_by(&krates) .into_iter() - .map(Keyword::into) - .collect::>(); + .map(TopVersions::from_versions) + .zip(data) + .map( + |(top_versions, (krate, total, recent, default_version, yanked))| { + Ok(EncodableCrate::from_minimal( + krate, + default_version.as_deref(), + yanked, + Some(&top_versions), + false, + total, + recent, + )) + }, + ) + .collect() + } - Ok(Json(json!({ - "num_downloads": num_downloads, - "num_crates": num_crates, - "new_crates": encode_crates(conn, new_crates)?, - "most_downloaded": encode_crates(conn, most_downloaded)?, - "most_recently_downloaded": encode_crates(conn, most_recently_downloaded)?, - "just_updated": encode_crates(conn, just_updated)?, - "popular_keywords": popular_keywords, - "popular_categories": popular_categories, - }))) - }) - .await + Ok(Json(json!({ + "num_downloads": num_downloads, + "num_crates": num_crates, + "new_crates": encode_crates(&mut conn, new_crates).await?, + "most_downloaded": encode_crates(&mut conn, most_downloaded).await?, + "most_recently_downloaded": encode_crates(&mut conn, most_recently_downloaded).await?, + "just_updated": encode_crates(&mut conn, just_updated).await?, + "popular_keywords": popular_keywords, + "popular_categories": popular_categories, + }))) } type Record = (Crate, i64, Option, Option, Option); From b4a14b99d29f8057626c7bf32b9b945bfe0d9a3e Mon Sep 17 00:00:00 2001 From: eth3lbert Date: Tue, 5 Nov 2024 07:44:51 +0800 Subject: [PATCH 5/5] controllers/summary: Inline changes --- src/controllers/summary.rs | 200 +++++++++++++++---------------------- 1 file changed, 82 insertions(+), 118 deletions(-) diff --git a/src/controllers/summary.rs b/src/controllers/summary.rs index 3bcf455832e..c38b65c44b8 100644 --- a/src/controllers/summary.rs +++ b/src/controllers/summary.rs @@ -6,8 +6,9 @@ use crate::schema::{ use crate::util::errors::AppResult; use crate::views::{EncodableCategory, EncodableCrate, EncodableKeyword}; use axum::Json; -use diesel::QueryResult; +use diesel::{ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper}; use diesel_async::AsyncPgConnection; +use diesel_async::RunQueryDsl; use serde_json::Value; /// Handles the `GET /summary` route. @@ -20,123 +21,11 @@ pub async fn summary(state: AppState) -> AppResult> { .map(Category::into) .collect::>(); - async fn inner( - conn: &mut AsyncPgConnection, - config: &crate::config::Server, - ) -> QueryResult<( - i64, - i64, - Vec, - Vec, - Vec, - Vec, - Vec, - )> { - use diesel::{ - ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper, - }; - use diesel_async::RunQueryDsl; - - let num_crates: i64 = crates::table.count().get_result(conn).await?; - let num_downloads: i64 = metadata::table - .select(metadata::total_downloads) - .get_result(conn) - .await?; - - let selection = ( - Crate::as_select(), - crate_downloads::downloads, - recent_crate_downloads::downloads.nullable(), - versions::num.nullable(), - versions::yanked.nullable(), - ); - - let new_crates = crates::table - .inner_join(crate_downloads::table) - .left_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .order(crates::created_at.desc()) - .select(selection) - .limit(10) - .load(conn) - .await?; - let just_updated = crates::table - .inner_join(crate_downloads::table) - .left_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .filter(crates::updated_at.ne(crates::created_at)) - .order(crates::updated_at.desc()) - .select(selection) - .limit(10) - .load(conn) - .await?; - - let mut most_downloaded_query = crates::table - .inner_join(crate_downloads::table) - .left_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .into_boxed(); - if !config.excluded_crate_names.is_empty() { - most_downloaded_query = - most_downloaded_query.filter(crates::name.ne_all(&config.excluded_crate_names)); - } - let most_downloaded = most_downloaded_query - .then_order_by(crate_downloads::downloads.desc()) - .select(selection) - .limit(10) - .load(conn) - .await?; - - let mut most_recently_downloaded_query = crates::table - .inner_join(crate_downloads::table) - .inner_join(recent_crate_downloads::table) - .left_join(default_versions::table) - .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) - .into_boxed(); - if !config.excluded_crate_names.is_empty() { - most_recently_downloaded_query = most_recently_downloaded_query - .filter(crates::name.ne_all(&config.excluded_crate_names)); - } - let most_recently_downloaded = most_recently_downloaded_query - .then_order_by(recent_crate_downloads::downloads.desc()) - .select(selection) - .limit(10) - .load(conn) - .await?; - - let popular_keywords = keywords::table - .order(keywords::crates_cnt.desc()) - .limit(10) - .load(conn) - .await? - .into_iter() - .map(Keyword::into) - .collect::>(); - - Ok(( - num_crates, - num_downloads, - new_crates, - just_updated, - most_downloaded, - most_recently_downloaded, - popular_keywords, - )) - } - - let config = &state.config; - let ( - num_crates, - num_downloads, - new_crates, - just_updated, - most_downloaded, - most_recently_downloaded, - popular_keywords, - ) = inner(&mut conn, config).await?; + let num_crates: i64 = crates::table.count().get_result(&mut conn).await?; + let num_downloads: i64 = metadata::table + .select(metadata::total_downloads) + .get_result(&mut conn) + .await?; async fn encode_crates( conn: &mut AsyncPgConnection, @@ -168,6 +57,81 @@ pub async fn summary(state: AppState) -> AppResult> { .collect() } + let config = &state.config; + + let selection = ( + Crate::as_select(), + crate_downloads::downloads, + recent_crate_downloads::downloads.nullable(), + versions::num.nullable(), + versions::yanked.nullable(), + ); + + let new_crates = crates::table + .inner_join(crate_downloads::table) + .left_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .order(crates::created_at.desc()) + .select(selection) + .limit(10) + .load(&mut conn) + .await?; + let just_updated = crates::table + .inner_join(crate_downloads::table) + .left_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .filter(crates::updated_at.ne(crates::created_at)) + .order(crates::updated_at.desc()) + .select(selection) + .limit(10) + .load(&mut conn) + .await?; + + let mut most_downloaded_query = crates::table + .inner_join(crate_downloads::table) + .left_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .into_boxed(); + if !config.excluded_crate_names.is_empty() { + most_downloaded_query = + most_downloaded_query.filter(crates::name.ne_all(&config.excluded_crate_names)); + } + let most_downloaded = most_downloaded_query + .then_order_by(crate_downloads::downloads.desc()) + .select(selection) + .limit(10) + .load(&mut conn) + .await?; + + let mut most_recently_downloaded_query = crates::table + .inner_join(crate_downloads::table) + .inner_join(recent_crate_downloads::table) + .left_join(default_versions::table) + .left_join(versions::table.on(default_versions::version_id.eq(versions::id))) + .into_boxed(); + if !config.excluded_crate_names.is_empty() { + most_recently_downloaded_query = most_recently_downloaded_query + .filter(crates::name.ne_all(&config.excluded_crate_names)); + } + let most_recently_downloaded = most_recently_downloaded_query + .then_order_by(recent_crate_downloads::downloads.desc()) + .select(selection) + .limit(10) + .load(&mut conn) + .await?; + + let popular_keywords = keywords::table + .order(keywords::crates_cnt.desc()) + .limit(10) + .load(&mut conn) + .await? + .into_iter() + .map(Keyword::into) + .collect::>(); + Ok(Json(json!({ "num_downloads": num_downloads, "num_crates": num_crates,