Merge pull request #166 from filecoin-project/feat/force-overwrite-te…

…mplate-file-in-allocator

Force overwrite template file in allocator

fplus-database/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "fplus-database"
 authors = ["clriesco", "kokal33", "alexmcon"]
-version = "1.7.2"
+version = "1.7.3"
 edition = "2021"
 description = "FPlus main database module"
 license = "MIT OR Apache-2.0"

fplus-http-server/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "fplus-http-server"
 authors = ["jbesraa", "kokal33", "clriesco"]
-version = "1.7.2"
+version = "1.7.3"
 description = "FPlus main http module"
 license = "MIT OR Apache-2.0"
 edition = "2021"

fplus-http-server/src/main.rs

@@ -98,6 +98,7 @@ async fn main() -> std::io::Result<()> {
             .service(router::allocator::delete)
             .service(router::allocator::create_from_json)
             .service(router::allocator::update_single_installation_id)
+            .service(router::allocator::update_allocator_force)
     })
     .bind(("0.0.0.0", 8080))?
     .run()

fplus-http-server/src/router/allocator.rs

@@ -1,8 +1,8 @@
 use actix_web::{get, post, put, delete, web, HttpResponse, Responder};
 use fplus_database::database::allocators as allocators_db;
-use fplus_lib::{core::{allocator::{
-    create_allocator_repo, is_allocator_repo_created, process_allocator_file, update_single_installation_id_logic
-}, AllocatorUpdateInfo, ChangedAllocators, InstallationIdUpdateInfo}, external_services::filecoin::get_multisig_threshold_for_actor};
+use fplus_lib::core::{allocator::{
+    create_allocator_repo, is_allocator_repo_created, process_allocator_file, update_single_installation_id_logic, force_update_allocators
+}, AllocatorUpdateInfo, ChangedAllocators, InstallationIdUpdateInfo, AllocatorUpdateForceInfo};
 
 /**
  * Get all allocators
@@ -206,4 +206,27 @@ pub async fn update_single_installation_id(query: web::Query<InstallationIdUpdat
             HttpResponse::InternalServerError().body(format!("{}", e))
         }
     }
+}
+
+/**
+ * Force updating allocator files from template.
+ * It receives a list of changed files and allocators to update.
+ * If allocators is not provided, it will update all allocators as long as they have an installation id.
+ * 
+ * # Arguments
+ * @param AllocatorUpdateForceInfo - The list of changed JSON file names and allocators to update
+ */
+#[post("/allocator/update/force")]
+pub async fn update_allocator_force(body: web::Json<AllocatorUpdateForceInfo>) -> impl Responder {
+    // First we need to deconstruct the body
+    let AllocatorUpdateForceInfo { files, allocators: affected_allocators } = body.into_inner();
+
+    // Logic will be implemented in allocator::update_allocator_force
+    match force_update_allocators(files, affected_allocators).await {
+        Ok(results) => HttpResponse::Ok().json(results),
+        Err(e) => {
+            log::error!("Failed to update allocators: {}", e);
+            HttpResponse::InternalServerError().body(format!("{}", e))
+        }
+    }
 }

fplus-lib/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "fplus-lib"
 authors = ["jbesraa", "kokal33", "clriesco"]
-version = "1.7.2"
+version = "1.7.3"
 edition = "2021"
 description = "FPlus library/helper files"
 license = "MIT OR Apache-2.0"
@@ -30,6 +30,6 @@ uuidv4 = "1.0.0"
 rayon = "1.8.0" 
 log = "0.4.20"
 once_cell = "1.19.0"
-fplus-database = { path = "../fplus-database", version = "1.7.0"}
+fplus-database = { path = "../fplus-database", version = "1.7.3"}
 pem = "1.0"
 anyhow = "1.0"

fplus-lib/src/core/allocator/mod.rs

@@ -1,6 +1,8 @@
-use fplus_database::database::allocators::create_or_update_allocator;
+use fplus_database::database::allocators::{create_or_update_allocator, get_allocators,};
+use fplus_database::models::allocators::Model;
+
 use octocrab::auth::create_jwt;
-use octocrab::models::repos::ContentItems;
+use octocrab::models::repos::{ContentItems, Content};
 
 use crate::config::get_env_var_or_default;
 use crate::external_services::filecoin::get_multisig_threshold_for_actor;
@@ -15,6 +17,8 @@ use jsonwebtoken::EncodingKey;
 use reqwest::{Client, header};
 use anyhow::Result;
 
+use super::GithubQueryParams;
+
 pub mod file;
 
 pub async fn process_allocator_file(file_name: &str) -> Result<AllocatorModel, LDNError> {
@@ -110,6 +114,52 @@ pub async fn is_allocator_repo_created(owner: &str, repo: &str) -> Result<bool,
     }
 }
 
+pub async fn create_file_in_repo(gh: &GithubWrapper, file: &Content, force: bool) -> Result<(), LDNError> {
+    let file_path = file.path.clone();
+    let file_sha = file.sha.clone();
+    let file = reqwest::Client::new()
+        .get(&file.download_url.clone().unwrap())
+        .send()
+        .await
+        .map_err(|e| LDNError::Load(format!("here {}", e)))?;
+    let file = file
+        .text()
+        .await
+        .map_err(|e| LDNError::Load(format!("here1 {}", e)))?;
+
+    //Get file from target repo. If file does not exist or fails to retrieve, create it
+    let target_file = gh.get_file(&file_path, "main").await.map_err(|e| {
+        LDNError::Load(format!("Failed to retrieve file from GitHub. Reason: {} in file {}", e, file_path))
+    });
+
+    match target_file {
+        Ok(target_file) => {
+            if target_file.items.is_empty() {
+                log::info!("Creating file in target repo: {}", file_path);
+                gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
+                    LDNError::Load(format!("Failed to create file in GitHub repo {}/{}. Reason: {} in file {}", gh.owner.clone(), gh.repo.clone(), e, file_path))
+                })?;
+            } else if !force {
+                log::info!("File already exists in target repo {}/{}: {}", gh.owner.clone(), gh.repo.clone(), file_path);
+            } else if target_file.items[0].sha.clone() != file_sha {
+                log::info!("Force creating file in target repo {}/{}: {}", gh.owner.clone(), gh.repo.clone(), file_path);
+                let file_sha = target_file.items[0].sha.clone();
+                gh.update_file(&file_path,"Update", &file, "main", &file_sha).await.map_err(|e| {
+                    LDNError::Load(format!("Failed to update file in GitHub repo {}/{}. Reason: {} in file {}", gh.owner.clone(), gh.repo.clone(), e, file_path))
+                })?;
+            }
+        },
+        Err(_) => {
+            log::info!("Creating file in target repo: {}", file_path);
+            gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
+                LDNError::Load(format!("Failed to create file in GitHub repo {}/{}. Reason: {} in file {}", gh.owner.clone(), gh.repo.clone(), e, file_path))
+            })?;
+        },
+    }
+
+    Ok(())
+}
+
 pub async fn create_allocator_repo(owner: &str, repo: &str) -> Result<(), LDNError> {
     let gh = github_async_new(owner.to_string(), repo.to_string()).await;
     let mut dirs = Vec::new();
@@ -133,39 +183,7 @@ pub async fn create_allocator_repo(owner: &str, repo: &str) -> Result<(), LDNErr
                 dirs.push(file_path);
                 continue;
             }
-            let file = reqwest::Client::new()
-            .get(&file.download_url.clone().unwrap())
-            .send()
-            .await
-            .map_err(|e| LDNError::Load(format!("here {}", e)))?;
-            let file = file
-                .text()
-                .await
-                .map_err(|e| LDNError::Load(format!("here1 {}", e)))?;
-
-            //Get file from target repo. If file does not exist or fails to retrieve, create it
-            let target_file = gh.get_file(&file_path, "main").await.map_err(|e| {
-                LDNError::Load(format!("Failed to retrieve file from GitHub. Reason: {} in file {}", e, file_path))
-            });
-
-            match target_file {
-                Ok(target_file) => {
-                    if target_file.items.is_empty() {
-                        log::info!("Creating file in target repo: {}", file_path);
-                        gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
-                            LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path))
-                        })?;
-                    } else {
-                        log::info!("File already exists in target repo: {}", file_path);
-                    }
-                },
-                Err(_) => {
-                    log::info!("Creating file in target repo: {}", file_path);
-                    gh.add_file(&file_path, &file, "first commit", "main").await.map_err(|e| {
-                        LDNError::Load(format!("Failed to create file in GitHub. Reason: {} in file {}", e, file_path))
-                    })?;
-                },
-            }
+            self::create_file_in_repo(&gh, &file, false).await?;
         }
     }
 
@@ -293,4 +311,76 @@ pub async fn update_single_installation_id_logic(installation_id: String) -> Res
     let installation = InstallationRepositories { installation_id: installation_id.parse().unwrap(), repositories };
     update_installation_ids_in_db(installation.clone()).await;
     return Ok(installation);
+}
+
+pub async fn force_update_allocators(files: Vec<String>, affected_allocators: Option<Vec<GithubQueryParams>>) -> Result<(), LDNError> {
+    // first get all allocators from db and filter by affected_allocators
+    let allocators = get_allocators()
+        .await
+        .map_err(|e| LDNError::Load(e.to_string()))?;
+
+    //filter allocators that have installation_id and msig_address
+    let allocators: Vec<Model> = allocators
+        .iter()
+        .filter(|a| a.installation_id.is_some() && a.multisig_address.is_some())
+        .cloned() // Clone the elements before collecting them
+        .collect();
+
+    // if affected_allocators is provided, filter allocators by owner and repo
+    let allocators: Vec<Model> = match affected_allocators {
+        Some(affected_allocators) => {
+            allocators
+                .iter()
+                .filter(|a| {
+                    affected_allocators
+                        .iter()
+                        .any(|aa| aa.owner == a.owner && aa.repo == a.repo)
+                })
+                .cloned()
+                .collect()
+        }
+        None => allocators,
+    };
+
+    //If no allocators return
+    if allocators.is_empty() {
+        log::info!("No allocators to update");
+        return Ok(());
+    }
+
+    let branch = match get_env_var_or_default("FILPLUS_ENV").as_str() {
+        "staging" => "staging",
+        "production" => "main",
+        _ => "main"
+    };
+
+    //now iterate over allocators and files
+    for allocator in allocators {
+        let gh = GithubWrapper::new(
+            allocator.owner, 
+            allocator.repo, 
+            allocator.installation_id.unwrap().to_string()
+        );
+
+        for file in files.iter() {
+
+            match gh.get_files_from_public_repo("fidlabs", "allocator-template", branch, Some(&file)).await {
+                Ok(content) => {
+                    match create_file_in_repo(&gh, &content.items[0], true).await {
+                        Ok(_) => {
+                            log::info!("File {} updated successfully", file);
+                        },
+                        Err(e) => {
+                            log::error!("{}", e);
+                        }
+                    }
+                },
+                Err(e) => {
+                    log::error!("{}", e);
+                }
+            }
+        }
+    }    
+
+    Ok(())
 }

fplus-lib/src/core/mod.rs

@@ -123,6 +123,12 @@ pub struct ChangedAllocators {
     pub files_changed: Vec<String>,
 }
 
+#[derive(Deserialize)]
+pub struct AllocatorUpdateForceInfo {
+    pub files: Vec<String>,
+    pub allocators: Option<Vec<GithubQueryParams>>
+}
+
 #[derive(Deserialize)]
 pub struct AllocatorUpdateInfo {
     pub installation_id: Option<i64>,

fplus-lib/src/external_services/github.rs

@@ -427,6 +427,24 @@ impl GithubWrapper {
         Ok(iid)
     }
 
+    pub async fn update_file (
+        &self,
+        path: &str,
+        message: &str,
+        content: &str,
+        branch: &str,
+        file_sha: &str,
+     ) -> Result<FileUpdate, OctocrabError> {
+        let iid = self
+            .inner
+            .repos(&self.owner, &self.repo)
+            .update_file(path, message, content, file_sha)
+            .branch(branch)
+            .send()
+            .await?;
+        Ok(iid)
+    }
+
     pub async fn get_pull_request_by_number(
         &self,
         number: u64,