Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 16 additions & 19 deletions atcoder-problems-backend/src/bin/lambda_batch_update.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
extern crate openssl; // Just for musl-compiler
use openssl_probe; // Just for musl-compiler

use atcoder_problems_backend::error::MapHandlerError;
use atcoder_problems_backend::sql::models::Submission;
use atcoder_problems_backend::sql::{
use atcoder_problems_backend as backend;

use backend::sql::connect;
use backend::sql::models::Submission;
use backend::sql::{
AcceptedCountClient, LanguageCountClient, ProblemInfoUpdater, ProblemsSubmissionUpdater,
RatedPointSumClient, StreakUpdater, SubmissionClient, SubmissionRequest,
};
use diesel::{Connection, PgConnection};

use lambda_runtime::error::HandlerError;
use lambda_runtime::lambda;
use lambda_runtime::Context;
Expand All @@ -28,42 +30,37 @@ fn handler(_: String, _: Context) -> Result<String, HandlerError> {

info!("Connecting to SQL ...");
let url = env::var("SQL_URL")?;
let conn = PgConnection::establish(&url).herr()?;
let conn = connect(&url)?;

info!("Loading submissions ...");
let all_accepted_submissions: Vec<Submission> = conn
.get_submissions(SubmissionRequest::AllAccepted)
.herr()?;
let all_accepted_submissions: Vec<Submission> =
conn.get_submissions(SubmissionRequest::AllAccepted)?;

info!(
"There are {} AC submissions.",
all_accepted_submissions.len()
);

info!("Executing update_accepted_count...");
conn.update_accepted_count(&all_accepted_submissions)
.herr()?;
conn.update_accepted_count(&all_accepted_submissions)?;

info!("Executing update_problem_solver_count...");
conn.update_solver_count().herr()?;
conn.update_solver_count()?;

info!("Executing update_rated_point_sums...");
conn.update_rated_point_sum(&all_accepted_submissions)
.herr()?;
conn.update_rated_point_sum(&all_accepted_submissions)?;

info!("Executing update_language_count...");
conn.update_language_count(&all_accepted_submissions)
.herr()?;
conn.update_language_count(&all_accepted_submissions)?;

info!("Executing update_submissions_of_problems...");
conn.update_submissions_of_problems(&all_accepted_submissions)
.herr()?;
conn.update_submissions_of_problems(&all_accepted_submissions)?;

info!("Executing update_problem_points...");
conn.update_problem_points().herr()?;
conn.update_problem_points()?;

info!("Executing update_streak_count...");
conn.update_streak_count(&all_accepted_submissions).herr()?;
conn.update_streak_count(&all_accepted_submissions)?;

info!("Finished");
Ok("Finished".to_owned())
Expand Down
12 changes: 6 additions & 6 deletions atcoder-problems-backend/src/crawler.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
use crate::error::Result;
use crate::sql::models::{Contest, ContestProblem, Problem, Submission};
use crate::sql::{ContestProblemClient, SimpleClient, SubmissionClient, SubmissionRequest};
use algorithm_problem_client::{AtCoderClient, AtCoderProblem};
use chrono::{Duration, Utc};
use diesel::QueryResult;
use log::{error, info};
use std::collections::BTreeSet;
use std::{thread, time};

const NEW_CONTEST_THRESHOLD_DAYS: i64 = 7;
const NEW_PAGE_THRESHOLD: usize = 5;

pub fn crawl_from_new_contests<C>(conn: &C, client: &AtCoderClient) -> QueryResult<()>
pub fn crawl_from_new_contests<C>(conn: &C, client: &AtCoderClient) -> Result<()>
where
C: SimpleClient + SubmissionClient,
{
Expand Down Expand Up @@ -53,7 +53,7 @@ where
Ok(())
}

pub fn crawl_all_submissions<C>(conn: &C, client: &AtCoderClient) -> QueryResult<()>
pub fn crawl_all_submissions<C>(conn: &C, client: &AtCoderClient) -> Result<()>
where
C: SimpleClient + SubmissionClient,
{
Expand All @@ -80,7 +80,7 @@ where
Ok(())
}

pub fn crawl_new_submissions<C>(conn: &C, client: &AtCoderClient) -> QueryResult<()>
pub fn crawl_new_submissions<C>(conn: &C, client: &AtCoderClient) -> Result<()>
where
C: SimpleClient + SubmissionClient,
{
Expand Down Expand Up @@ -113,7 +113,7 @@ where
Ok(())
}

pub fn crawl_from_recent_submitted<C>(conn: &C, client: &AtCoderClient) -> QueryResult<()>
pub fn crawl_from_recent_submitted<C>(conn: &C, client: &AtCoderClient) -> Result<()>
where
C: SimpleClient + SubmissionClient,
{
Expand Down Expand Up @@ -147,7 +147,7 @@ where
Ok(())
}

pub fn crawl_contest_and_problems<C>(conn: &C, client: &AtCoderClient) -> QueryResult<()>
pub fn crawl_contest_and_problems<C>(conn: &C, client: &AtCoderClient) -> Result<()>
where
C: SimpleClient + ContestProblemClient,
{
Expand Down
36 changes: 27 additions & 9 deletions atcoder-problems-backend/src/error.rs
Original file line number Diff line number Diff line change
@@ -1,18 +1,36 @@
use diesel::{ConnectionResult, QueryResult};
use lambda_runtime::error::HandlerError;
use serde::export::Formatter;

pub trait MapHandlerError<T> {
fn herr(self) -> Result<T, HandlerError>;
#[derive(Debug)]
pub enum Error {
ConnectionError(diesel::ConnectionError),
QueryError(diesel::result::Error),
}

impl<T> MapHandlerError<T> for ConnectionResult<T> {
fn herr(self) -> Result<T, HandlerError> {
self.map_err(|e| HandlerError::from(e.to_string().as_str()))
impl From<diesel::ConnectionError> for Error {
fn from(e: diesel::ConnectionError) -> Self {
Error::ConnectionError(e)
}
}

impl<T> MapHandlerError<T> for QueryResult<T> {
fn herr(self) -> Result<T, HandlerError> {
self.map_err(|e| HandlerError::from(e.to_string().as_str()))
impl From<diesel::result::Error> for Error {
fn from(e: diesel::result::Error) -> Self {
Error::QueryError(e)
}
}

pub type Result<T> = std::result::Result<T, Error>;

impl From<Error> for HandlerError {
fn from(e: Error) -> Self {
HandlerError::from(e.to_string().as_str())
}
}

impl std::error::Error for Error {}

impl std::fmt::Display for Error {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "({:?})", self)
}
}
4 changes: 1 addition & 3 deletions atcoder-problems-backend/src/lambda/time_submissions.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use super::{LambdaInput, LambdaOutput};
use crate::error::MapHandlerError;
use crate::sql::{SubmissionClient, SubmissionRequest};

use diesel::{Connection, ConnectionResult, PgConnection};
Expand Down Expand Up @@ -34,8 +33,7 @@ where
.get_submissions(SubmissionRequest::FromTime {
from_second: from_epoch_second,
count: 1000,
})
.herr()?;
})?;
let max_id = submissions.iter().map(|s| s.id).max().unwrap_or(0);

let mut hasher = Md5::new();
Expand Down
14 changes: 9 additions & 5 deletions atcoder-problems-backend/src/lambda/user_info.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
use super::{LambdaInput, LambdaOutput};
use crate::error::MapHandlerError;
use crate::error::Result;
use crate::sql::{AcceptedCountClient, RatedPointSumClient};

use diesel::{Connection, ConnectionResult, PgConnection, QueryResult};
use diesel::{Connection, ConnectionResult, PgConnection};
use lambda_runtime::{error::HandlerError, Context, Handler};
use log::info;
use serde::Serialize;

fn get_user_info<'a, C>(conn: &C, user_id: &'a str) -> QueryResult<UserInfo<'a>>
fn get_user_info<'a, C>(conn: &C, user_id: &'a str) -> Result<UserInfo<'a>>
where
C: AcceptedCountClient + RatedPointSumClient,
{
Expand Down Expand Up @@ -48,13 +48,17 @@ impl<C> Handler<LambdaInput, LambdaOutput, HandlerError> for UserInfoHandler<C>
where
C: AcceptedCountClient + RatedPointSumClient,
{
fn run(&mut self, e: LambdaInput, _: Context) -> Result<LambdaOutput, HandlerError> {
fn run(
&mut self,
e: LambdaInput,
_: Context,
) -> std::result::Result<LambdaOutput, HandlerError> {
let user_id = e
.param("user")
.ok_or_else(|| HandlerError::from("There is no user."))?;

info!("UserInfo API");
let user_info = get_user_info(&self.connection, user_id).herr()?;
let user_info = get_user_info(&self.connection, user_id)?;

let body = serde_json::to_string(&user_info)?;
Ok(LambdaOutput::new200(body, None))
Expand Down
6 changes: 2 additions & 4 deletions atcoder-problems-backend/src/lambda/user_submissions.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use super::{LambdaInput, LambdaOutput};
use crate::error::MapHandlerError;
use crate::sql::{SubmissionClient, SubmissionRequest};

use diesel::{Connection, ConnectionResult, PgConnection};
Expand Down Expand Up @@ -28,7 +27,7 @@ where
let user_id = e
.param("user")
.ok_or_else(|| HandlerError::from("There is no user."))?;
let count: i64 = self.connection.get_user_submission_count(user_id).herr()?;
let count: i64 = self.connection.get_user_submission_count(user_id)?;

let mut hasher = Md5::new();
hasher.input(user_id.as_bytes());
Expand All @@ -41,8 +40,7 @@ where
_ => {
let submissions = self
.connection
.get_submissions(SubmissionRequest::UserAll { user_id })
.herr()?;
.get_submissions(SubmissionRequest::UserAll { user_id })?;
let body = serde_json::to_string(&submissions)?;
Ok(LambdaOutput::new200(body, Some(etag)))
}
Expand Down
8 changes: 8 additions & 0 deletions atcoder-problems-backend/src/sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,11 @@ pub use rated_point_sum::RatedPointSumClient;
pub use simple_client::SimpleClient;
pub use streak::StreakUpdater;
pub use submission_client::{SubmissionClient, SubmissionRequest};

use crate::error::Result;
use diesel::{Connection, PgConnection};

pub fn connect(url: &str) -> Result<PgConnection> {
let connection = PgConnection::establish(url)?;
Ok(connection)
}
34 changes: 19 additions & 15 deletions atcoder-problems-backend/src/sql/accepted_count.rs
Original file line number Diff line number Diff line change
@@ -1,43 +1,47 @@
use super::MAX_INSERT_ROWS;
use crate::error::Result;
use crate::sql::models::{Submission, UserProblemCount};
use crate::sql::schema::accepted_count;
use crate::utils::SplitToSegments;
use diesel::dsl::*;
use diesel::pg::upsert::excluded;
use diesel::prelude::*;
use diesel::{insert_into, PgConnection, QueryResult};
use diesel::{insert_into, PgConnection};
use std::collections::{BTreeMap, BTreeSet};

pub trait AcceptedCountClient {
fn load_accepted_count(&self) -> QueryResult<Vec<UserProblemCount>>;
fn get_users_accepted_count(&self, user_id: &str) -> QueryResult<i32>;
fn get_accepted_count_rank(&self, accepted_count: i32) -> QueryResult<i64>;
fn update_accepted_count(&self, submissions: &[Submission]) -> QueryResult<()>;
fn load_accepted_count(&self) -> Result<Vec<UserProblemCount>>;
fn get_users_accepted_count(&self, user_id: &str) -> Result<i32>;
fn get_accepted_count_rank(&self, accepted_count: i32) -> Result<i64>;
fn update_accepted_count(&self, submissions: &[Submission]) -> Result<()>;
}

impl AcceptedCountClient for PgConnection {
fn load_accepted_count(&self) -> QueryResult<Vec<UserProblemCount>> {
accepted_count::table
fn load_accepted_count(&self) -> Result<Vec<UserProblemCount>> {
let count = accepted_count::table
.order_by(accepted_count::problem_count.desc())
.then_order_by(accepted_count::user_id.asc())
.load::<UserProblemCount>(self)
.load::<UserProblemCount>(self)?;
Ok(count)
}

fn get_users_accepted_count(&self, user_id: &str) -> QueryResult<i32> {
accepted_count::table
fn get_users_accepted_count(&self, user_id: &str) -> Result<i32> {
let count = accepted_count::table
.filter(accepted_count::user_id.eq(user_id))
.select(accepted_count::problem_count)
.first::<i32>(self)
.first::<i32>(self)?;
Ok(count)
}

fn get_accepted_count_rank(&self, accepted_count: i32) -> QueryResult<i64> {
accepted_count::table
fn get_accepted_count_rank(&self, accepted_count: i32) -> Result<i64> {
let rank = accepted_count::table
.filter(accepted_count::problem_count.gt(accepted_count))
.select(count_star())
.first::<i64>(self)
.first::<i64>(self)?;
Ok(rank)
}

fn update_accepted_count(&self, submissions: &[Submission]) -> QueryResult<()> {
fn update_accepted_count(&self, submissions: &[Submission]) -> Result<()> {
let accepted_count = submissions
.iter()
.map(|s| (s.user_id.as_str(), s.problem_id.as_str()))
Expand Down
19 changes: 11 additions & 8 deletions atcoder-problems-backend/src/sql/contest_problem.rs
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
use crate::error::Result;
use crate::sql::models::ContestProblem;
use crate::sql::schema::contest_problem;

use diesel::dsl::*;
use diesel::prelude::*;
use diesel::{PgConnection, QueryResult};
use diesel::PgConnection;

pub trait ContestProblemClient {
fn insert_contest_problem(&self, contest_problems: &[ContestProblem]) -> QueryResult<usize>;
fn load_contest_problem(&self) -> QueryResult<Vec<ContestProblem>>;
fn insert_contest_problem(&self, contest_problems: &[ContestProblem]) -> Result<usize>;
fn load_contest_problem(&self) -> Result<Vec<ContestProblem>>;
}

impl ContestProblemClient for PgConnection {
fn insert_contest_problem(&self, contest_problems: &[ContestProblem]) -> QueryResult<usize> {
insert_into(contest_problem::table)
fn insert_contest_problem(&self, contest_problems: &[ContestProblem]) -> Result<usize> {
let result = insert_into(contest_problem::table)
.values(contest_problems)
.on_conflict((contest_problem::contest_id, contest_problem::problem_id))
.do_nothing()
.execute(self)
.execute(self)?;
Ok(result)
}

fn load_contest_problem(&self) -> QueryResult<Vec<ContestProblem>> {
contest_problem::table.load::<ContestProblem>(self)
fn load_contest_problem(&self) -> Result<Vec<ContestProblem>> {
let problems = contest_problem::table.load::<ContestProblem>(self)?;
Ok(problems)
}
}
Loading