feat(folders): move folders/files metadata out of Folders entries

sn_cli/src/subcommands/acc_packet.rs

@@ -0,0 +1,380 @@
+// Copyright 2024 MaidSafe.net limited.
+//
+// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3.
+// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed
+// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. Please review the Licences for the specific language governing
+// permissions and limitations relating to use of the SAFE Network Software.
+
+use super::files::{download_file, upload_files, ChunkManager, UploadedFile, UPLOADED_FILES};
+
+use serde::{Deserialize, Serialize};
+use sn_client::{Client, FilesApi, FolderEntry, FoldersApi, Metadata, WalletClient, BATCH_SIZE};
+use sn_protocol::storage::{Chunk, ChunkAddress, RegisterAddress, RetryStrategy};
+use sn_transfers::HotWallet;
+
+use color_eyre::{
+    eyre::{bail, eyre},
+    Result,
+};
+use std::{
+    collections::BTreeMap,
+    ffi::OsString,
+    fs::{create_dir_all, File},
+    io::Write,
+    path::{Path, PathBuf},
+};
+use tokio::task::JoinSet;
+use walkdir::WalkDir;
+use xor_name::XorName;
+
+// Name of hidden folder where tracking information and metadata is locally kept.
+const SAFE_TRACKING_CHANGES_DIR: &str = ".safe";
+
+// Subfolder where chunks will be cached
+const CHUNKS_CACHE_DIR: &str = "chunks";
+
+// Subfolder where files metadata will be cached
+const METADATA_CACHE_DIR: &str = "metadata";
+
+#[derive(Debug, Serialize, Deserialize)]
+struct TrackingMetadata {
+    file_path: PathBuf,
+    metadata: Metadata,
+}
+
+pub struct AccountPacket {
+    client: Client,
+    wallet_dir: PathBuf,
+    //entries_metadata: BTreeMap<EntryHash, Metadata>,
+    files_dir: PathBuf,
+    meta_dir: PathBuf,
+    chunks_dir: PathBuf,
+}
+
+impl AccountPacket {
+    /// Create AccountPacket instance.
+    pub fn new(client: Client, wallet_dir: &Path, path: &Path) -> Result<Self> {
+        let files_dir = path.to_path_buf().canonicalize()?;
+        let track_changes_dir = files_dir.join(SAFE_TRACKING_CHANGES_DIR);
+        let meta_dir = track_changes_dir.join(METADATA_CACHE_DIR);
+        let chunks_dir = track_changes_dir.join(CHUNKS_CACHE_DIR);
+        create_dir_all(&meta_dir)?;
+        create_dir_all(&chunks_dir)?;
+
+        Ok(Self {
+            client,
+            wallet_dir: wallet_dir.to_path_buf(),
+            files_dir,
+            meta_dir,
+            chunks_dir,
+        })
+    }
+
+    pub async fn add_all_files(&mut self) -> Result<RegisterAddress> {
+        let make_public = false;
+        let verify_store = true;
+
+        upload_files(
+            self.files_dir.clone(),
+            make_public,
+            &self.client,
+            self.wallet_dir.clone(),
+            verify_store,
+            10, //batch_size,
+            RetryStrategy::Quick,
+        )
+        .await?;
+
+        let mut chunk_manager = ChunkManager::new(&self.wallet_dir);
+        chunk_manager.chunk_path(&self.files_dir, true, make_public)?;
+
+        let mut folders = self.build_folders_hierarchy().await?;
+
+        // add chunked files to the corresponding Folders
+        for chunked_file in chunk_manager.iter_chunked_files() {
+            if let Some(parent) = chunked_file.file_path.parent() {
+                if let Some(folder) = folders.get_mut(parent) {
+                    let (metadata, meta_xorname) = folder
+                        .add_file(
+                            chunked_file.file_name.clone(),
+                            chunked_file.head_chunk_address,
+                        )
+                        .await?;
+
+                    self.write_metadata(&chunked_file.file_path, metadata, meta_xorname)?;
+                }
+            }
+        }
+
+        println!("Paying for folders hierarchy and uploading...");
+        let root_dir_address = folders
+            .get(&self.files_dir)
+            .map(|folder| *folder.address())
+            .ok_or(eyre!("Failed to obtain main Folder network address"))?;
+
+        self.pay_and_upload_folders(folders, verify_store).await?;
+
+        Ok(root_dir_address)
+    }
+
+    pub async fn download_folders(
+        &self,
+        address: RegisterAddress,
+        folder_name: OsString,
+        download_path: &Path,
+    ) -> Result<()> {
+        let mut files_to_download = vec![];
+        let mut folders_to_download = vec![(folder_name, address)];
+
+        while let Some((name, folder_addr)) = folders_to_download.pop() {
+            if !name.is_empty() {
+                println!(
+                    "Downloading Folder {name:?} from {}",
+                    hex::encode(folder_addr.xorname())
+                );
+            }
+            self.download_folder(
+                download_path,
+                address,
+                &mut files_to_download,
+                &mut folders_to_download,
+            )
+            .await?;
+        }
+
+        let files_api: FilesApi = FilesApi::new(self.client.clone(), download_path.to_path_buf());
+        // FIXME: wallet_dir is currently root_dir, but this will change
+        let uploaded_files_path = self.wallet_dir.join(UPLOADED_FILES);
+        for (file_name, addr, path) in files_to_download {
+            // try to read the data_map if it exists locally.
+            let expected_data_map_location = uploaded_files_path.join(addr.to_hex());
+            let local_data_map = UploadedFile::read(&expected_data_map_location)
+                .map(|uploaded_file_metadata| {
+                    uploaded_file_metadata.data_map.map(|bytes| Chunk {
+                        address: ChunkAddress::new(*addr.xorname()),
+                        value: bytes,
+                    })
+                })
+                .unwrap_or(None);
+
+            download_file(
+                files_api.clone(),
+                *addr.xorname(),
+                (file_name, local_data_map),
+                &path,
+                false,
+                10,                   //batch_size,
+                RetryStrategy::Quick, //retry_strategy,
+            )
+            .await;
+        }
+
+        Ok(())
+    }
+
+    pub async fn status(&mut self) -> Result<()> {
+        let make_public = false;
+        let verify_store = true;
+
+        upload_files(
+            self.files_dir.clone(),
+            make_public,
+            &self.client,
+            self.wallet_dir.clone(),
+            verify_store,
+            10, //batch_size,
+            RetryStrategy::Quick,
+        )
+        .await?;
+
+        let mut chunk_manager = ChunkManager::new(&self.wallet_dir);
+        chunk_manager.chunk_path(&self.files_dir, true, make_public)?;
+
+        let mut folders = self.build_folders_hierarchy().await?;
+
+        // add chunked files to the corresponding Folders
+        for chunked_file in chunk_manager.iter_chunked_files() {
+            if let Some(parent) = chunked_file.file_path.parent() {
+                if let Some(folder) = folders.get_mut(parent) {
+                    println!(
+                        ">> FILE {:?} -> {}",
+                        chunked_file.file_path,
+                        hex::encode(chunked_file.head_chunk_address.xorname())
+                    );
+                    let (metadata, meta_xorname) = folder
+                        .add_file(
+                            chunked_file.file_name.clone(),
+                            chunked_file.head_chunk_address,
+                        )
+                        .await?;
+
+                    let metadata_file = self.meta_dir.join(hex::encode(meta_xorname));
+                    if metadata_file.exists() {
+                        println!(
+                            "META FOUND for {:?} >> {metadata_file:?}",
+                            chunked_file.file_name
+                        );
+                    } else {
+                        println!(
+                            "NOT FOUND for {:?} >> {metadata_file:?}",
+                            chunked_file.file_name
+                        );
+                    }
+
+                    //self.write_metadata(&chunked_file.file_path, metadata, meta_xorname)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    // Private helpers
+
+    // Store metadata in a file to keep track of any changes made to the source file/folder
+    fn write_metadata(
+        &self,
+        src_path: &Path,
+        metadata: Metadata,
+        meta_xorname: XorName,
+    ) -> Result<()> {
+        let metadata_file = self.meta_dir.join(hex::encode(meta_xorname));
+        println!(">>META>> {metadata_file:?} -> {metadata:?}");
+        let mut meta_file = File::create(&metadata_file)?;
+
+        let file_path = src_path
+            .to_path_buf()
+            .canonicalize()?
+            .strip_prefix(&self.files_dir)?
+            .to_path_buf();
+        let tracking_meta = TrackingMetadata {
+            file_path,
+            metadata,
+        };
+        //meta_file.write_all(&rmp_serde::to_vec(&tracking_meta)?)?;
+        meta_file.write_all(format!("{tracking_meta:?}").as_bytes())?;
+        Ok(())
+    }
+
+    // Build Folders hierarchy from the set files dir
+    async fn build_folders_hierarchy(&self) -> Result<BTreeMap<PathBuf, FoldersApi>> {
+        let mut folders = BTreeMap::new();
+        for (dir_path, depth, parent, dir_name) in WalkDir::new(&self.files_dir)
+            .into_iter()
+            .filter_entry(|e| e.file_type().is_dir() && e.file_name() != SAFE_TRACKING_CHANGES_DIR)
+            .flatten()
+            .filter_map(|entry| {
+                entry.path().parent().map(|parent| {
+                    (
+                        entry.path().to_path_buf(),
+                        entry.depth(),
+                        parent.to_owned(),
+                        entry.file_name().to_owned(),
+                    )
+                })
+            })
+        {
+            let curr_folder_addr = *folders
+                .entry(dir_path.clone())
+                .or_insert(FoldersApi::new(self.client.clone(), &self.wallet_dir)?)
+                .address();
+
+            if depth > 0 {
+                let parent_folder = folders
+                    .entry(parent)
+                    .or_insert(FoldersApi::new(self.client.clone(), &self.wallet_dir)?);
+                let (metadata, meta_xorname) =
+                    parent_folder.add_folder(dir_name, curr_folder_addr).await?;
+                self.write_metadata(&dir_path, metadata, meta_xorname)?;
+            }
+        }
+
+        Ok(folders)
+    }
+
+    // Make a single payment for all Folders (Registers) and upload them to the network
+    async fn pay_and_upload_folders(
+        &self,
+        folders: BTreeMap<PathBuf, FoldersApi>,
+        verify_store: bool,
+    ) -> Result<()> {
+        // Let's make the storage payment
+        let mut wallet_client =
+            WalletClient::new(self.client.clone(), HotWallet::load_from(&self.wallet_dir)?);
+        let net_addresses = folders.values().map(|folder| folder.as_net_addr());
+        let payment_result = wallet_client.pay_for_storage(net_addresses).await?;
+        let balance = wallet_client.balance();
+        match payment_result
+            .storage_cost
+            .checked_add(payment_result.royalty_fees)
+        {
+            Some(cost) => println!(
+                "Made payment of {cost} for {} Folders. New balance: {balance}",
+                folders.len()
+            ),
+            None => bail!("Failed to calculate total payment cost"),
+        }
+
+        // sync Folders concurrently
+        let mut tasks = JoinSet::new();
+        for (path, mut folder) in folders {
+            let net_addr = folder.as_net_addr();
+            let payment = wallet_client.get_payment_for_addr(&net_addr)?;
+            let payment_info = payment_result
+                .payee_map
+                .get(&net_addr)
+                .map(|payee| (payment, *payee));
+
+            tasks.spawn(async move {
+                match folder.sync(verify_store, payment_info).await {
+                    Ok(addr) => println!(
+                        "Folder (for {}) synced with the network at: {}",
+                        path.display(),
+                        addr.to_hex()
+                    ),
+                    Err(err) => println!(
+                        "Failed to sync Folder (for {}) with the network: {err}",
+                        path.display(),
+                    ),
+                }
+            });
+        }
+
+        while let Some(res) = tasks.join_next().await {
+            if let Err(err) = res {
+                println!("Failed to sync a Folder with the network: {err:?}");
+            }
+        }
+
+        Ok(())
+    }
+
+    // Download a Folder from the network and keep track of its subfolders and files
+    async fn download_folder(
+        &self,
+        target_path: &Path,
+        folder_addr: RegisterAddress,
+        files_to_download: &mut Vec<(OsString, ChunkAddress, PathBuf)>,
+        folders_to_download: &mut Vec<(OsString, RegisterAddress)>,
+    ) -> Result<()> {
+        create_dir_all(target_path)?;
+        let folders_api =
+            FoldersApi::retrieve(self.client.clone(), &self.wallet_dir, folder_addr).await?;
+
+        for Metadata { name, content } in folders_api.entries().await?.into_iter() {
+            match content {
+                FolderEntry::File(file_addr) => files_to_download.push((
+                    name.clone().into(),
+                    file_addr,
+                    target_path.to_path_buf(),
+                )),
+                FolderEntry::Folder(subfolder_addr) => {
+                    folders_to_download.push((name.clone().into(), subfolder_addr));
+                }
+            }
+        }
+
+        Ok(())
+    }
+}