diff --git a/atcoder-problems-backend/Dockerfile b/atcoder-problems-backend/Dockerfile index 1d6cb89e3..7da091316 100644 --- a/atcoder-problems-backend/Dockerfile +++ b/atcoder-problems-backend/Dockerfile @@ -11,3 +11,4 @@ FROM rust:1.38 COPY --from=builder /app/target/release/crawl /usr/bin/crawl COPY --from=builder /app/target/release/dump_json /usr/bin/dump_json COPY --from=builder /app/target/release/delta_update /usr/bin/delta_update +COPY --from=builder /app/target/release/update /usr/bin/update diff --git a/atcoder-problems-backend/src/bin/lambda_delta_update.rs b/atcoder-problems-backend/src/bin/lambda_delta_update.rs deleted file mode 100644 index f744096a5..000000000 --- a/atcoder-problems-backend/src/bin/lambda_delta_update.rs +++ /dev/null @@ -1,71 +0,0 @@ -extern crate openssl; - -use atcoder_problems_backend::error::MapHandlerError; -use atcoder_problems_backend::sql::{ - AcceptedCountClient, LanguageCountClient, RatedPointSumClient, StreakUpdater, SubmissionClient, - SubmissionRequest, -}; -use diesel::{Connection, PgConnection}; -use lambda_runtime::{error::HandlerError, lambda, Context}; -use log::{self, info}; -use openssl_probe; -use simple_logger; -use std::collections::BTreeSet; -use std::env; -use std::error::Error; - -fn main() -> Result<(), Box> { - simple_logger::init_with_level(log::Level::Info)?; - openssl_probe::init_ssl_cert_env_vars(); - info!("Started!"); - lambda!(handler); - - Ok(()) -} - -fn handler(_: String, _: Context) -> Result { - let url = env::var("SQL_URL")?; - let conn = PgConnection::establish(&url).map_handler_error()?; - - info!("Connected"); - - let request = SubmissionRequest::RecentAccepted { count: 1000 }; - let recent_submissions = conn.get_submissions(request).map_handler_error()?; - info!("There are {} submissions.", recent_submissions.len()); - - let user_ids = recent_submissions - .into_iter() - .map(|s| s.user_id) - .collect::>(); - let user_ids = user_ids.iter().map(|s| s.as_str()).collect::>(); - info!("There are {} users to update", user_ids.len()); - - let request = SubmissionRequest::UsersAccepted { - user_ids: &user_ids, - }; - let user_accepted_submissions = conn.get_submissions(request).map_handler_error()?; - info!( - "There are {} submissions to use.", - user_accepted_submissions.len() - ); - - info!("Executing update_rated_point_sum..."); - conn.update_rated_point_sum(&user_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_accepted_count..."); - conn.update_accepted_count(&user_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_language_count..."); - conn.update_language_count(&user_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_streak_count..."); - conn.update_streak_count(&user_accepted_submissions) - .map_handler_error()?; - - info!("Finished"); - - Ok("Finished".to_owned()) -} diff --git a/atcoder-problems-backend/src/bin/lambda_dumper.rs b/atcoder-problems-backend/src/bin/lambda_dumper.rs deleted file mode 100644 index bcd682646..000000000 --- a/atcoder-problems-backend/src/bin/lambda_dumper.rs +++ /dev/null @@ -1,145 +0,0 @@ -extern crate openssl; - -use atcoder_problems_backend::error::MapHandlerError; -use atcoder_problems_backend::s3; -use atcoder_problems_backend::sql::models::*; -use atcoder_problems_backend::sql::schema::*; -use atcoder_problems_backend::sql::LanguageCountClient; -use diesel::prelude::*; -use diesel::{sql_query, Connection, PgConnection}; -use lambda_runtime::{error::HandlerError, lambda, Context}; -use log::{self, info}; -use openssl_probe; -use serde::Serialize; -use serde_json; -use simple_logger; -use std::env; -use std::error::Error; - -fn main() -> Result<(), Box> { - simple_logger::init_with_level(log::Level::Info)?; - openssl_probe::init_ssl_cert_env_vars(); - lambda!(handler); - Ok(()) -} - -fn handler(_: String, _: Context) -> Result<(), HandlerError> { - info!("Started!"); - let url = env::var("SQL_URL")?; - let conn: PgConnection = PgConnection::establish(&url).map_handler_error()?; - - let merged_query = sql_query(r" - SELECT - problems.id, - problems.contest_id, - problems.title, - shortest.submission_id AS shortest_submission_id, - shortest.problem_id AS shortest_problem_id, - shortest.contest_id AS shortest_contest_id, - shortest_submissions.user_id AS shortest_user_id, - fastest.submission_id AS fastest_submission_id, - fastest.problem_id AS fastest_problem_id, - fastest.contest_id AS fastest_contest_id, - fastest_submissions.user_id AS fastest_user_id, - first.submission_id AS first_submission_id, - first.problem_id AS first_problem_id, - first.contest_id AS first_contest_id, - first_submissions.user_id AS first_user_id, - shortest_submissions.length AS source_code_length, - fastest_submissions.execution_time AS execution_time, - points.point, - points.predict, - solver.user_count AS solver_count - FROM - problems - LEFT JOIN shortest ON shortest.problem_id = problems.id - LEFT JOIN fastest ON fastest.problem_id = problems.id - LEFT JOIN first ON first.problem_id = problems.id - LEFT JOIN submissions AS shortest_submissions ON shortest.submission_id = shortest_submissions.id - LEFT JOIN submissions AS fastest_submissions ON fastest.submission_id = fastest_submissions.id - LEFT JOIN submissions AS first_submissions ON first.submission_id = first_submissions.id - LEFT JOIN points ON points.problem_id = problems.id - LEFT JOIN solver ON solver.problem_id = problems.id - ORDER BY problems.id; - "); - - let data_paths = vec![ - ( - merged_query - .load::(&conn) - .serialize_to_bytes()?, - "resources/merged-problems.json", - ), - ( - contests::table - .order_by(contests::id) - .load::(&conn) - .serialize_to_bytes()?, - "resources/contests.json", - ), - ( - accepted_count::table - .order_by(accepted_count::user_id) - .load::(&conn) - .serialize_to_bytes()?, - "resources/ac.json", - ), - ( - problems::table - .order_by(problems::id) - .load::(&conn) - .serialize_to_bytes()?, - "resources/problems.json", - ), - ( - rated_point_sum::table - .order_by(rated_point_sum::user_id) - .load::(&conn) - .serialize_to_bytes()?, - "resources/sums.json", - ), - ( - conn.load_language_count().serialize_to_bytes()?, - "resources/lang.json", - ), - ( - contest_problem::table - .order_by(contest_problem::problem_id) - .load::(&conn) - .serialize_to_bytes()?, - "resources/contest-problem.json", - ), - ( - max_streaks::table - .order_by(max_streaks::user_id) - .load::(&conn) - .serialize_to_bytes()?, - "resources/streaks.json", - ), - ]; - - let client = s3::S3Client::new(); - for (data, path) in data_paths.into_iter() { - info!("Uploading {}", path); - client.update(data, path, s3::ContentType::Json)?; - info!("Uploaded"); - } - - info!("Done."); - Ok(()) -} - -trait SerializeToBytes { - fn serialize_to_bytes(self) -> Result, HandlerError>; -} - -impl SerializeToBytes for QueryResult -where - T: Serialize, -{ - fn serialize_to_bytes(self) -> Result, HandlerError> { - let result = self.map_handler_error()?; - let vec = serde_json::to_vec(&result)?; - Ok(vec) - } -} diff --git a/atcoder-problems-backend/src/bin/lambda_update.rs b/atcoder-problems-backend/src/bin/lambda_update.rs deleted file mode 100644 index 56c5db638..000000000 --- a/atcoder-problems-backend/src/bin/lambda_update.rs +++ /dev/null @@ -1,18 +0,0 @@ -extern crate openssl; - -use atcoder_problems_backend::lambda::DatabaseUpdateHandler; - -use lambda_runtime; -use log; -use openssl_probe; -use simple_logger; -use std::env; -use std::error::Error; - -fn main() -> Result<(), Box> { - simple_logger::init_with_level(log::Level::Info)?; - openssl_probe::init_ssl_cert_env_vars(); - let url = env::var("SQL_URL")?; - lambda_runtime::start(DatabaseUpdateHandler::new(&url)?, None); - Ok(()) -} diff --git a/atcoder-problems-backend/src/bin/update.rs b/atcoder-problems-backend/src/bin/update.rs new file mode 100644 index 000000000..dc8332844 --- /dev/null +++ b/atcoder-problems-backend/src/bin/update.rs @@ -0,0 +1,58 @@ +extern crate openssl; // Just for musl-compiler +use openssl_probe; // Just for musl-compiler + +use atcoder_problems_backend::sql::models::Submission; +use atcoder_problems_backend::sql::{ + AcceptedCountClient, LanguageCountClient, ProblemInfoUpdater, ProblemsSubmissionUpdater, + RatedPointSumClient, StreakUpdater, SubmissionClient, SubmissionRequest, +}; +use diesel::{Connection, PgConnection}; +use log::{self, info}; +use simple_logger; +use std::env; +use std::error::Error; + +fn main() -> Result<(), Box> { + openssl_probe::init_ssl_cert_env_vars(); // Just for musl-compiler + + simple_logger::init_with_level(log::Level::Info)?; + info!("Started!"); + + info!("Connecting to SQL ..."); + let url = env::var("SQL_URL")?; + let conn = PgConnection::establish(&url)?; + + info!("Loading submissions ..."); + let all_accepted_submissions: Vec = + conn.get_submissions(SubmissionRequest::AllAccepted)?; + + info!( + "There are {} AC submissions.", + all_accepted_submissions.len() + ); + + info!("Executing update_accepted_count..."); + conn.update_accepted_count(&all_accepted_submissions)?; + + info!("Executing update_problem_solver_count..."); + conn.update_solver_count()?; + + info!("Executing update_rated_point_sums..."); + conn.update_rated_point_sum(&all_accepted_submissions)?; + + info!("Executing update_language_count..."); + conn.update_language_count(&all_accepted_submissions)?; + + info!("Executing update_submissions_of_problems..."); + conn.update_submissions_of_problems(&all_accepted_submissions)?; + + info!("Executing update_problem_points..."); + conn.update_problem_points()?; + + info!("Executing update_streak_count..."); + conn.update_streak_count(&all_accepted_submissions)?; + + info!("Finished"); + + Ok(()) +} diff --git a/atcoder-problems-backend/src/lambda.rs b/atcoder-problems-backend/src/lambda.rs index 79d23521e..9228d86e5 100644 --- a/atcoder-problems-backend/src/lambda.rs +++ b/atcoder-problems-backend/src/lambda.rs @@ -1,10 +1,8 @@ -mod database_update; mod io; mod time_submissions; mod user_info; mod user_submissions; -pub use database_update::DatabaseUpdateHandler; pub(crate) use io::{LambdaInput, LambdaOutput}; pub use time_submissions::TimeSubmissionsHandler; pub use user_info::UserInfoHandler; diff --git a/atcoder-problems-backend/src/lambda/database_update.rs b/atcoder-problems-backend/src/lambda/database_update.rs deleted file mode 100644 index 6e103767a..000000000 --- a/atcoder-problems-backend/src/lambda/database_update.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::error::MapHandlerError; -use crate::sql::models::Submission; -use crate::sql::{ - AcceptedCountClient, LanguageCountClient, ProblemInfoUpdater, ProblemsSubmissionUpdater, - RatedPointSumClient, StreakUpdater, SubmissionClient, SubmissionRequest, -}; -use diesel::{Connection, ConnectionResult, PgConnection}; -use lambda_runtime::{error::HandlerError, Context, Handler}; -use log::info; - -pub struct DatabaseUpdateHandler { - connection: C, -} - -impl DatabaseUpdateHandler { - pub fn new(url: &str) -> ConnectionResult { - let connection = PgConnection::establish(&url)?; - Ok(Self { connection }) - } -} - -impl Handler for DatabaseUpdateHandler -where - C: AcceptedCountClient - + LanguageCountClient - + ProblemsSubmissionUpdater - + RatedPointSumClient - + SubmissionClient - + ProblemInfoUpdater - + StreakUpdater, -{ - fn run(&mut self, _: String, _: Context) -> Result { - let all_accepted_submissions: Vec = self - .connection - .get_submissions(SubmissionRequest::AllAccepted) - .map_handler_error()?; - - info!( - "There are {} AC submissions.", - all_accepted_submissions.len() - ); - - info!("Executing update_accepted_count..."); - self.connection - .update_accepted_count(&all_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_problem_solver_count..."); - self.connection.update_solver_count().map_handler_error()?; - - info!("Executing update_rated_point_sums..."); - self.connection - .update_rated_point_sum(&all_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_language_count..."); - self.connection - .update_language_count(&all_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_submissions_of_problems..."); - self.connection - .update_submissions_of_problems(&all_accepted_submissions) - .map_handler_error()?; - - info!("Executing update_problem_points..."); - self.connection - .update_problem_points() - .map_handler_error()?; - - info!("Executing update_streak_count..."); - self.connection - .update_streak_count(&all_accepted_submissions) - .map_handler_error()?; - - Ok("Finished".to_owned()) - } -}