-
Notifications
You must be signed in to change notification settings - Fork 35
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(folders): move folders/files metadata out of Folders entries
- Loading branch information
Showing
6 changed files
with
494 additions
and
251 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,380 @@ | ||
// Copyright 2024 MaidSafe.net limited. | ||
// | ||
// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. | ||
// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed | ||
// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
// KIND, either express or implied. Please review the Licences for the specific language governing | ||
// permissions and limitations relating to use of the SAFE Network Software. | ||
|
||
use super::files::{download_file, upload_files, ChunkManager, UploadedFile, UPLOADED_FILES}; | ||
|
||
use serde::{Deserialize, Serialize}; | ||
use sn_client::{Client, FilesApi, FolderEntry, FoldersApi, Metadata, WalletClient, BATCH_SIZE}; | ||
use sn_protocol::storage::{Chunk, ChunkAddress, RegisterAddress, RetryStrategy}; | ||
use sn_transfers::HotWallet; | ||
|
||
use color_eyre::{ | ||
eyre::{bail, eyre}, | ||
Result, | ||
}; | ||
use std::{ | ||
collections::BTreeMap, | ||
ffi::OsString, | ||
fs::{create_dir_all, File}, | ||
io::Write, | ||
path::{Path, PathBuf}, | ||
}; | ||
use tokio::task::JoinSet; | ||
use walkdir::WalkDir; | ||
use xor_name::XorName; | ||
|
||
// Name of hidden folder where tracking information and metadata is locally kept. | ||
const SAFE_TRACKING_CHANGES_DIR: &str = ".safe"; | ||
|
||
// Subfolder where chunks will be cached | ||
const CHUNKS_CACHE_DIR: &str = "chunks"; | ||
|
||
// Subfolder where files metadata will be cached | ||
const METADATA_CACHE_DIR: &str = "metadata"; | ||
|
||
// Information stored locally to keep track of local changes to files/folders. | ||
// TODO: to make file changes discovery more efficient, add more info like file size and last modified timestamp. | ||
Check notice Code scanning / devskim A "TODO" or similar was left in source code, possibly indicating incomplete functionality Note
Suspicious comment
|
||
#[derive(Debug, Serialize, Deserialize)] | ||
struct TrackingMetadata { | ||
file_path: PathBuf, | ||
metadata: Metadata, | ||
} | ||
|
||
pub struct AccountPacket { | ||
client: Client, | ||
wallet_dir: PathBuf, | ||
//entries_metadata: BTreeMap<EntryHash, Metadata>, | ||
files_dir: PathBuf, | ||
meta_dir: PathBuf, | ||
chunks_dir: PathBuf, | ||
} | ||
|
||
impl AccountPacket { | ||
/// Create AccountPacket instance. | ||
pub fn new(client: Client, wallet_dir: &Path, path: &Path) -> Result<Self> { | ||
let files_dir = path.to_path_buf().canonicalize()?; | ||
let track_changes_dir = files_dir.join(SAFE_TRACKING_CHANGES_DIR); | ||
let meta_dir = track_changes_dir.join(METADATA_CACHE_DIR); | ||
let chunks_dir = track_changes_dir.join(CHUNKS_CACHE_DIR); | ||
create_dir_all(&meta_dir)?; | ||
create_dir_all(&chunks_dir)?; | ||
|
||
Ok(Self { | ||
client, | ||
wallet_dir: wallet_dir.to_path_buf(), | ||
files_dir, | ||
meta_dir, | ||
chunks_dir, | ||
}) | ||
} | ||
|
||
pub async fn add_all_files(&mut self) -> Result<RegisterAddress> { | ||
let make_public = false; | ||
let verify_store = true; | ||
|
||
upload_files( | ||
self.files_dir.clone(), | ||
make_public, | ||
&self.client, | ||
self.wallet_dir.clone(), | ||
verify_store, | ||
10, //batch_size, | ||
RetryStrategy::Quick, | ||
) | ||
.await?; | ||
|
||
let mut chunk_manager = ChunkManager::new(&self.wallet_dir); | ||
chunk_manager.chunk_path(&self.files_dir, true, make_public)?; | ||
|
||
let mut folders = self.build_folders_hierarchy().await?; | ||
|
||
// add chunked files to the corresponding Folders | ||
for chunked_file in chunk_manager.iter_chunked_files() { | ||
if let Some(parent) = chunked_file.file_path.parent() { | ||
if let Some(folder) = folders.get_mut(parent) { | ||
let (metadata, meta_xorname) = folder | ||
.add_file( | ||
chunked_file.file_name.clone(), | ||
chunked_file.head_chunk_address, | ||
) | ||
.await?; | ||
|
||
self.write_tracking_metadata(&chunked_file.file_path, metadata, meta_xorname)?; | ||
} | ||
} | ||
} | ||
|
||
println!("Paying for folders hierarchy and uploading..."); | ||
let root_dir_address = folders | ||
.get(&self.files_dir) | ||
.map(|folder| *folder.address()) | ||
.ok_or(eyre!("Failed to obtain main Folder network address"))?; | ||
|
||
self.pay_and_upload_folders(folders, verify_store).await?; | ||
|
||
Ok(root_dir_address) | ||
} | ||
|
||
pub async fn download_folders( | ||
&self, | ||
address: RegisterAddress, | ||
folder_name: OsString, | ||
download_path: &Path, | ||
) -> Result<()> { | ||
let mut files_to_download = vec![]; | ||
let mut folders_to_download = vec![(folder_name, address)]; | ||
|
||
while let Some((name, folder_addr)) = folders_to_download.pop() { | ||
if !name.is_empty() { | ||
println!( | ||
"Downloading Folder {name:?} from {}", | ||
hex::encode(folder_addr.xorname()) | ||
); | ||
} | ||
self.download_folder( | ||
download_path, | ||
address, | ||
&mut files_to_download, | ||
&mut folders_to_download, | ||
) | ||
.await?; | ||
} | ||
|
||
let files_api: FilesApi = FilesApi::new(self.client.clone(), download_path.to_path_buf()); | ||
// FIXME: wallet_dir is currently root_dir, but this will change and chunk mgr will allow different paths. | ||
Check notice Code scanning / devskim A "TODO" or similar was left in source code, possibly indicating incomplete functionality Note
Suspicious comment
|
||
let uploaded_files_path = self.wallet_dir.join(UPLOADED_FILES); | ||
for (file_name, addr, path) in files_to_download { | ||
// try to read the data_map if it exists locally. | ||
let expected_data_map_location = uploaded_files_path.join(addr.to_hex()); | ||
let local_data_map = UploadedFile::read(&expected_data_map_location) | ||
.map(|uploaded_file_metadata| { | ||
uploaded_file_metadata.data_map.map(|bytes| Chunk { | ||
address: ChunkAddress::new(*addr.xorname()), | ||
value: bytes, | ||
}) | ||
}) | ||
.unwrap_or(None); | ||
|
||
download_file( | ||
files_api.clone(), | ||
*addr.xorname(), | ||
(file_name, local_data_map), | ||
&path, | ||
false, | ||
10, //batch_size, | ||
RetryStrategy::Quick, //retry_strategy, | ||
) | ||
.await; | ||
} | ||
|
||
Ok(()) | ||
} | ||
|
||
pub async fn status(&mut self) -> Result<()> { | ||
let make_public = false; | ||
let verify_store = true; | ||
|
||
upload_files( | ||
self.files_dir.clone(), | ||
make_public, | ||
&self.client, | ||
self.wallet_dir.clone(), | ||
verify_store, | ||
10, //batch_size, | ||
RetryStrategy::Quick, | ||
) | ||
.await?; | ||
|
||
let mut chunk_manager = ChunkManager::new(&self.wallet_dir); | ||
chunk_manager.chunk_path(&self.files_dir, true, make_public)?; | ||
|
||
let mut folders = self.build_folders_hierarchy().await?; | ||
|
||
// add chunked files to the corresponding Folders | ||
for chunked_file in chunk_manager.iter_chunked_files() { | ||
if let Some(parent) = chunked_file.file_path.parent() { | ||
if let Some(folder) = folders.get_mut(parent) { | ||
println!( | ||
">> FILE {:?} -> {}", | ||
chunked_file.file_path, | ||
hex::encode(chunked_file.head_chunk_address.xorname()) | ||
); | ||
let (metadata, meta_xorname) = folder | ||
.add_file( | ||
chunked_file.file_name.clone(), | ||
chunked_file.head_chunk_address, | ||
) | ||
.await?; | ||
|
||
let metadata_file = self.meta_dir.join(hex::encode(meta_xorname)); | ||
if metadata_file.exists() { | ||
println!( | ||
"META FOUND for {:?} >> {metadata_file:?}", | ||
chunked_file.file_name | ||
); | ||
} else { | ||
println!( | ||
"NOT FOUND for {:?} >> {metadata_file:?}", | ||
chunked_file.file_name | ||
); | ||
} | ||
} | ||
} | ||
} | ||
|
||
Ok(()) | ||
} | ||
|
||
// Private helpers | ||
|
||
// Store tracking metadata in a file to keep track of any changes made to the source file/folder | ||
fn write_tracking_metadata( | ||
&self, | ||
src_path: &Path, | ||
metadata: Metadata, | ||
meta_xorname: XorName, | ||
) -> Result<()> { | ||
let metadata_file = self.meta_dir.join(hex::encode(meta_xorname)); | ||
println!(">>META>> {metadata_file:?} -> {metadata:?}"); | ||
let mut meta_file = File::create(&metadata_file)?; | ||
|
||
let file_path = src_path | ||
.to_path_buf() | ||
.canonicalize()? | ||
.strip_prefix(&self.files_dir)? | ||
.to_path_buf(); | ||
let tracking_meta = TrackingMetadata { | ||
file_path, | ||
metadata, | ||
}; | ||
//meta_file.write_all(&rmp_serde::to_vec(&tracking_meta)?)?; | ||
meta_file.write_all(format!("{tracking_meta:?}").as_bytes())?; | ||
Ok(()) | ||
} | ||
|
||
// Build Folders hierarchy from the set files dir | ||
async fn build_folders_hierarchy(&self) -> Result<BTreeMap<PathBuf, FoldersApi>> { | ||
let mut folders = BTreeMap::new(); | ||
for (dir_path, depth, parent, dir_name) in WalkDir::new(&self.files_dir) | ||
.into_iter() | ||
.filter_entry(|e| e.file_type().is_dir() && e.file_name() != SAFE_TRACKING_CHANGES_DIR) | ||
.flatten() | ||
.filter_map(|entry| { | ||
entry.path().parent().map(|parent| { | ||
( | ||
entry.path().to_path_buf(), | ||
entry.depth(), | ||
parent.to_owned(), | ||
entry.file_name().to_owned(), | ||
) | ||
}) | ||
}) | ||
{ | ||
let curr_folder_addr = *folders | ||
.entry(dir_path.clone()) | ||
.or_insert(FoldersApi::new(self.client.clone(), &self.wallet_dir)?) | ||
.address(); | ||
|
||
if depth > 0 { | ||
let parent_folder = folders | ||
.entry(parent) | ||
.or_insert(FoldersApi::new(self.client.clone(), &self.wallet_dir)?); | ||
let (metadata, meta_xorname) = | ||
parent_folder.add_folder(dir_name, curr_folder_addr).await?; | ||
self.write_tracking_metadata(&dir_path, metadata, meta_xorname)?; | ||
} | ||
} | ||
|
||
Ok(folders) | ||
} | ||
|
||
// Make a single payment for all Folders (Registers) and upload them to the network | ||
async fn pay_and_upload_folders( | ||
&self, | ||
folders: BTreeMap<PathBuf, FoldersApi>, | ||
verify_store: bool, | ||
) -> Result<()> { | ||
// Let's make the storage payment | ||
let mut wallet_client = | ||
WalletClient::new(self.client.clone(), HotWallet::load_from(&self.wallet_dir)?); | ||
let net_addresses = folders.values().map(|folder| folder.as_net_addr()); | ||
let payment_result = wallet_client.pay_for_storage(net_addresses).await?; | ||
let balance = wallet_client.balance(); | ||
match payment_result | ||
.storage_cost | ||
.checked_add(payment_result.royalty_fees) | ||
{ | ||
Some(cost) => println!( | ||
"Made payment of {cost} for {} Folders. New balance: {balance}", | ||
folders.len() | ||
), | ||
None => bail!("Failed to calculate total payment cost"), | ||
} | ||
|
||
// sync Folders concurrently | ||
let mut tasks = JoinSet::new(); | ||
for (path, mut folder) in folders { | ||
let net_addr = folder.as_net_addr(); | ||
let payment = wallet_client.get_payment_for_addr(&net_addr)?; | ||
let payment_info = payment_result | ||
.payee_map | ||
.get(&net_addr) | ||
.map(|payee| (payment, *payee)); | ||
|
||
tasks.spawn(async move { | ||
match folder.sync(verify_store, payment_info).await { | ||
Ok(addr) => println!( | ||
"Folder (for {}) synced with the network at: {}", | ||
path.display(), | ||
addr.to_hex() | ||
), | ||
Err(err) => println!( | ||
"Failed to sync Folder (for {}) with the network: {err}", | ||
path.display(), | ||
), | ||
} | ||
}); | ||
} | ||
|
||
while let Some(res) = tasks.join_next().await { | ||
if let Err(err) = res { | ||
println!("Failed to sync a Folder with the network: {err:?}"); | ||
} | ||
} | ||
|
||
Ok(()) | ||
} | ||
|
||
// Download a Folder from the network and keep track of its subfolders and files | ||
async fn download_folder( | ||
&self, | ||
target_path: &Path, | ||
folder_addr: RegisterAddress, | ||
files_to_download: &mut Vec<(OsString, ChunkAddress, PathBuf)>, | ||
folders_to_download: &mut Vec<(OsString, RegisterAddress)>, | ||
) -> Result<()> { | ||
create_dir_all(target_path)?; | ||
let folders_api = | ||
FoldersApi::retrieve(self.client.clone(), &self.wallet_dir, folder_addr).await?; | ||
|
||
for Metadata { name, content } in folders_api.entries().await?.into_iter() { | ||
match content { | ||
FolderEntry::File(file_addr) => files_to_download.push(( | ||
name.clone().into(), | ||
file_addr, | ||
target_path.to_path_buf(), | ||
)), | ||
FolderEntry::Folder(subfolder_addr) => { | ||
folders_to_download.push((name.clone().into(), subfolder_addr)); | ||
} | ||
} | ||
} | ||
|
||
Ok(()) | ||
} | ||
} |
Oops, something went wrong.