From 104f06db631f40f31bde63e355fe0a84ffed545e Mon Sep 17 00:00:00 2001 From: Sauravcv98 <65706094+Sauravcv98@users.noreply.github.com> Date: Mon, 27 May 2024 11:48:27 +0530 Subject: [PATCH] feat: reducing context tool (#44) * fix: reduce context tool added * fix: comments fixes 1 * fix: comments fixes 2 * fix: comments fixes 3 --------- Co-authored-by: Saurav CV --- .../src/api/config/handlers.rs | 354 +++++++++++++++++- .../src/api/context/handlers.rs | 29 +- .../src/api/context/mod.rs | 5 + 3 files changed, 369 insertions(+), 19 deletions(-) diff --git a/crates/context_aware_config/src/api/config/handlers.rs b/crates/context_aware_config/src/api/config/handlers.rs index 73b6bc2e..85e70ecc 100644 --- a/crates/context_aware_config/src/api/config/handlers.rs +++ b/crates/context_aware_config/src/api/config/handlers.rs @@ -4,13 +4,23 @@ use std::{collections::HashMap, str::FromStr}; use super::helpers::{ filter_config_by_dimensions, filter_config_by_prefix, filter_context, }; - -use super::types::Config; -use crate::db::schema::{ - contexts::dsl as ctxt, default_configs::dsl as def_conf, event_log::dsl as event_log, +use super::types::{Config, Context}; +use crate::api::context::{ + delete_context_api, hash, put, validate_dimensions_and_calculate_priority, PutReq, +}; +use crate::api::dimension::get_all_dimension_schema_map; +use crate::{ + db::schema::{ + contexts::dsl as ctxt, default_configs::dsl as def_conf, + event_log::dsl as event_log, + }, + helpers::json_to_sorted_string, }; use actix_http::header::{HeaderName, HeaderValue}; -use actix_web::{get, web::Query, HttpRequest, HttpResponse, Scope}; +use actix_web::web; +use actix_web::{ + error::ErrorBadRequest, get, put, web::Query, HttpRequest, HttpResponse, Scope, +}; use cac_client::{eval_cac, eval_cac_with_reasoning, MergeStrategy}; use chrono::{DateTime, NaiveDateTime, TimeZone, Timelike, Utc}; use diesel::{ @@ -22,13 +32,17 @@ use serde_json::{json, Map, Value}; use service_utils::service::types::DbConnection; use service_utils::{bad_argument, db_error, unexpected_error}; -use service_utils::result as superposition; +use itertools::Itertools; +use jsonschema::JSONSchema; +use service_utils::helpers::extract_dimensions; +use service_utils::result::{self as superposition, AppError}; +use superposition_types::User; use uuid::Uuid; - pub fn endpoints() -> Scope { Scope::new("") .service(get) .service(get_resolved_config) + .service(reduce_config) .service(get_filtered_config) } @@ -147,6 +161,332 @@ async fn generate_cac( }) } +fn generate_subsets(map: &Map) -> Vec> { + let mut subsets = Vec::new(); + let keys: Vec = map.keys().cloned().collect_vec(); + let all_subsets_keys = generate_subsets_keys(keys); + + for subset_keys in &all_subsets_keys { + if subset_keys.len() >= 0 { + let mut subset_map = Map::new(); + + for key in subset_keys { + if let Some(value) = map.get(key) { + subset_map.insert(key.to_string(), value.clone()); + } + } + + subsets.push(subset_map); + } + } + + subsets +} + +fn generate_subsets_keys(keys: Vec) -> Vec> { + let mut res = vec![[].to_vec()]; + for element in keys { + let len = res.len(); + for ind in 0..len { + let mut sub = res[ind].clone(); + sub.push(element.clone()); + res.push(sub); + } + } + res +} + +fn reduce( + contexts_overrides_values: Vec<(Context, Map, Value, String)>, + default_config_val: &Value, +) -> superposition::Result>> { + let mut dimensions: Vec> = Vec::new(); + for (context, overrides, key_val, override_id) in contexts_overrides_values { + let mut ct_dimensions = extract_dimensions(&context.condition)?; + ct_dimensions.insert("key_val".to_string(), key_val); + let request_payload = json!({ + "override": overrides, + "context": context.condition, + "id": context.id, + "to_be_deleted": overrides.is_empty(), + "override_id": override_id, + }); + ct_dimensions.insert("req_payload".to_string(), request_payload); + dimensions.push(ct_dimensions); + } + + //adding default config value + let mut default_config_map = Map::new(); + default_config_map.insert("key_val".to_string(), default_config_val.to_owned()); + dimensions.push(default_config_map); + + /* + We now have dimensions array, which is a vector of elements representing each context present where each element is a type of Map which contains the following + 1. all the dimensions and value of those dimensions in the context + 2. key_val, which is the value of the override key for which we are trying to reduce + 3. A req_payload which contains the details of the context like, context_id, override_id, the context_condition, new overrides (without containing the key that has to be reduced) + { + dimension1_in_context : value_of_dimension1_in_context, + dimension2_in_context : value_of_dimension2_in_context, + . + . + key_val: value of the override key that we are trying to reduce + req_payload : { + override : new_overrides(without the key that is to be reduced) + context : context_condition + id : context_id + to_be_deleted : if new_overrides is empty then delete this context + } + } + + We have also sorted this dimensions vector in descending order based on the priority of the dimensions in that context + and in this vector the default config will be at the end of the list as it has no dimensions and it's priority is the least + + Now we iterate from start and then pick an element and generate all subsets of that element keys excluding the req_payload and key_val + i.e we only generate different subsets of dimensions of that context along with the value of those dimensions in that context + + Next we check if in the vector we find any other element c2 whose dimensions is part of the subsets of the parent element c1 + if dimensions_subsets_of_c1 contains dimensions_of_c2 + + if the value of the override key is same in both c1 and c2 then we can reduce or remove that key in the override of c1 + so we mark the can_be_reduce to be true, and then update the dimensions vector. + + but if we find any other element c3 whose dimensions is a subset of c1_dimensions but the value is not the same + then that means we can't reduce this key from c1, because in resolve if we remove it from c1 it will pick the value form c3 which is different. + So if we find this element c3 before any other element which is a subset of c1 with the same value, then we can't reduce this key for c1 so we break + and continue with the next element. + Here "before" means the element with higher priority comes first with a subset of c1 but differnt override value for the key + */ + for (c1_index, dimensions_of_c1_with_payload) in dimensions.clone().iter().enumerate() + { + let mut dimensions_of_c1 = dimensions_of_c1_with_payload.clone(); + dimensions_of_c1.remove("req_payload"); + let override_val_of_key_in_c1 = dimensions_of_c1.remove("key_val"); + let dimensions_subsets_of_c1 = generate_subsets(&dimensions_of_c1); + for (c2_index, dimensions_in_c2_with_payload) in dimensions.iter().enumerate() { + let mut dimensions_of_c2 = dimensions_in_c2_with_payload.clone(); + dimensions_of_c2.remove("req_payload"); + let override_val_of_key_in_c2 = dimensions_of_c2.remove("key_val"); + if c2_index != c1_index { + if dimensions_subsets_of_c1.contains(&dimensions_of_c2) { + if override_val_of_key_in_c1 == override_val_of_key_in_c2 { + let mut temp_c1 = dimensions_of_c1_with_payload.to_owned(); + temp_c1.insert("can_be_reduced".to_string(), Value::Bool(true)); + dimensions[c1_index] = temp_c1; + break; + } else if override_val_of_key_in_c2.is_some() { + break; + } + } + } + } + } + Ok(dimensions) +} + +fn get_contextids_from_overrideid( + contexts: Vec, + overrides: Map, + key_val: Value, + override_id: &str, +) -> superposition::Result, Value, String)>> { + let mut res: Vec<(Context, Map, Value, String)> = Vec::new(); + for ct in contexts { + let ct_dimensions = extract_dimensions(&ct.condition)?; + if ct_dimensions.contains_key("variantIds") { + continue; + } + let override_keys = &ct.override_with_keys; + if override_keys.contains(&override_id.to_owned()) { + res.push(( + ct, + overrides.clone(), + key_val.clone(), + override_id.to_string(), + )); + } + } + Ok(res) +} + +fn construct_new_payload(req_payload: &Map) -> web::Json { + let mut res = req_payload.clone(); + res.remove("to_be_deleted"); + res.remove("override_id"); + res.remove("id"); + if let Some(Value::Object(res_context)) = res.get("context") { + if let Some(Value::Object(res_override)) = res.get("override") { + return web::Json(PutReq { + context: res_context.to_owned(), + r#override: res_override.to_owned(), + }); + } + } + web::Json(PutReq { + context: Map::new(), + r#override: Map::new(), + }) +} + +async fn reduce_config_key( + user: User, + conn: &mut PooledConnection>, + mut og_contexts: Vec, + mut og_overrides: Map, + check_key: &str, + dimension_schema_map: &HashMap, + default_config: Map, + is_approve: bool, +) -> superposition::Result { + let default_config_val = + default_config + .get(check_key) + .ok_or(AppError::BadArgument(format!( + "{} not found in default config", + check_key + )))?; + let mut contexts_overrides_values = Vec::new(); + + for (override_id, override_value) in og_overrides.clone() { + match override_value { + Value::Object(mut override_obj) => { + if let Some(value_of_check_key) = override_obj.remove(check_key) { + let context_arr = get_contextids_from_overrideid( + og_contexts.clone(), + override_obj, + value_of_check_key.clone(), + &override_id, + )?; + contexts_overrides_values.extend(context_arr); + } + } + _ => (), + } + } + + let mut priorities = Vec::new(); + + for (index, ctx) in contexts_overrides_values.iter().enumerate() { + let priority = validate_dimensions_and_calculate_priority( + "context", + &(ctx.0).condition, + dimension_schema_map, + )?; + priorities.push((index, priority)) + } + + // Sort the collected results based on priority + priorities.sort_by(|a, b| b.1.cmp(&a.1)); + + // Use the sorted indices to reorder the original vector + let sorted_priority_contexts = priorities + .into_iter() + .map(|(index, _)| contexts_overrides_values[index].clone()) + .collect(); + + let resolved_dimensions = reduce(sorted_priority_contexts, default_config_val)?; + for rd in resolved_dimensions { + match ( + rd.get("can_be_reduced"), + rd.get("req_payload"), + rd.get("req_payload").and_then(|v| v.get("id")), + rd.get("req_payload").and_then(|v| v.get("override_id")), + rd.get("req_payload").and_then(|v| v.get("to_be_deleted")), + rd.get("req_payload").and_then(|v| v.get("override")), + ) { + ( + Some(Value::Bool(true)), + Some(Value::Object(request_payload)), + Some(Value::String(cid)), + Some(Value::String(oid)), + Some(Value::Bool(to_be_deleted)), + Some(override_val), + ) => { + if *to_be_deleted { + if is_approve { + let _ = delete_context_api(cid.clone(), user.clone(), conn).await; + } + og_contexts.retain(|x| x.id != *cid); + } else { + if is_approve { + let _ = delete_context_api(cid.clone(), user.clone(), conn).await; + let put_req = construct_new_payload(request_payload); + let _ = put(put_req, conn, false, &user); + } + + let new_id = hash(override_val); + og_overrides.insert(new_id.clone(), override_val.clone()); + + let mut ctx_index = 0; + let mut delete_old_oid = true; + + for (ind, ctx) in og_contexts.iter().enumerate() { + if ctx.id == *cid { + ctx_index = ind; + } else if ctx.override_with_keys.contains(oid) { + delete_old_oid = false; + } + } + + let mut elem = og_contexts[ctx_index].clone(); + elem.override_with_keys = [new_id]; + og_contexts[ctx_index] = elem; + + if delete_old_oid { + og_overrides.remove(oid); + } + } + } + _ => continue, + } + } + + Ok(Config { + contexts: og_contexts, + overrides: og_overrides, + default_configs: default_config, + }) +} + +#[put("/reduce")] +async fn reduce_config( + req: HttpRequest, + user: User, + db_conn: DbConnection, +) -> superposition::Result { + let DbConnection(mut conn) = db_conn; + let is_approve = req + .headers() + .get("x-approve") + .and_then(|value| value.to_str().ok().and_then(|s| s.parse::().ok())) + .unwrap_or(false); + + let dimensions_schema_map = get_all_dimension_schema_map(&mut conn)?; + let mut config = generate_cac(&mut conn).await?; + let default_config = (config.default_configs).clone(); + for (key, val) in default_config { + let contexts = config.contexts; + let overrides = config.overrides; + let default_config = config.default_configs; + config = reduce_config_key( + user.clone(), + &mut conn, + contexts.clone(), + overrides.clone(), + key.as_str(), + &dimensions_schema_map, + default_config.clone(), + is_approve, + ) + .await?; + if is_approve { + config = generate_cac(&mut conn).await?; + } + } + + Ok(HttpResponse::Ok().json(config)) +} + #[get("")] async fn get( req: HttpRequest, diff --git a/crates/context_aware_config/src/api/context/handlers.rs b/crates/context_aware_config/src/api/context/handlers.rs index 1ff58c27..2700b56d 100644 --- a/crates/context_aware_config/src/api/context/handlers.rs +++ b/crates/context_aware_config/src/api/context/handlers.rs @@ -61,7 +61,7 @@ pub fn endpoints() -> Scope { type DBConnection = PooledConnection>; -fn validate_dimensions_and_calculate_priority( +pub fn validate_dimensions_and_calculate_priority( object_key: &str, cond: &Value, dimension_schema_map: &HashMap, @@ -217,7 +217,7 @@ fn create_ctx_from_put_req( }) } -fn hash(val: &Value) -> String { +pub fn hash(val: &Value) -> String { let sorted_str: String = json_to_sorted_string(val); blake3::hash(sorted_str.as_bytes()).to_string() } @@ -272,7 +272,7 @@ fn get_put_resp(ctx: Context) -> PutResp { } } -fn put( +pub fn put( req: Json, conn: &mut PooledConnection>, already_under_txn: bool, @@ -493,18 +493,13 @@ async fn list_contexts( Ok(Json(result)) } -#[delete("/{ctx_id}")] -async fn delete_context( - path: Path, - db_conn: DbConnection, +pub async fn delete_context_api( + ctx_id: String, user: User, + conn: &mut PooledConnection>, ) -> superposition::Result { use contexts::dsl; - let DbConnection(mut conn) = db_conn; - - let ctx_id = path.into_inner(); - let deleted_row = - delete(dsl::contexts.filter(dsl::id.eq(&ctx_id))).execute(&mut conn); + let deleted_row = delete(dsl::contexts.filter(dsl::id.eq(&ctx_id))).execute(conn); match deleted_row { Ok(0) => Err(not_found!("Context Id `{}` doesn't exists", ctx_id)), Ok(_) => { @@ -518,6 +513,16 @@ async fn delete_context( } } +#[delete("/{ctx_id}")] +async fn delete_context( + path: Path, + user: User, + mut db_conn: DbConnection, +) -> superposition::Result { + let ctx_id = path.into_inner(); + delete_context_api(ctx_id, user, &mut db_conn).await +} + #[put("/bulk-operations")] async fn bulk_operations( reqs: Json>, diff --git a/crates/context_aware_config/src/api/context/mod.rs b/crates/context_aware_config/src/api/context/mod.rs index 8c255ad4..6b9ea678 100644 --- a/crates/context_aware_config/src/api/context/mod.rs +++ b/crates/context_aware_config/src/api/context/mod.rs @@ -1,4 +1,9 @@ mod handlers; pub mod helpers; mod types; +pub use handlers::delete_context_api; pub use handlers::endpoints; +pub use handlers::hash; +pub use handlers::put; +pub use handlers::validate_dimensions_and_calculate_priority; +pub use types::PutReq;