From 209ded3d80286a994d4a55b753b1e6ac63ee6487 Mon Sep 17 00:00:00 2001 From: Kira Sotnikov Date: Fri, 11 Apr 2025 09:57:45 +0300 Subject: [PATCH] WIP Cli tool for read WT files --- Cargo.toml | 4 +- .../persist_table/generator/space_file/mod.rs | 61 ++++++++++++------- .../generator/space_file/worktable_impls.rs | 11 ++-- codegen/src/worktable/generator/row.rs | 29 ++++++++- examples/src/main.rs | 12 ++-- src/lib.rs | 2 +- src/row.rs | 2 + 7 files changed, 84 insertions(+), 37 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 372ff8a7..df0d2261 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,8 +25,8 @@ lockfree = { version = "0.5.1" } worktable_codegen = { path = "codegen", version = "0.5.1" } futures = "0.3.30" uuid = { version = "1.10.0", features = ["v4"] } -data_bucket = { git = "https://github.com/pathscale/DataBucket", rev = "3eb4fc2" } -#data_bucket = { path = "../DataBucket", version = "0.2.1" } +#data_bucket = { git = "https://github.com/pathscale/DataBucket", rev = "3eb4fc2" } +data_bucket = { path = "../DataBucket", version = "0.2.1" } performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true } indexset = { version = "0.11.3", features = ["concurrent", "cdc", "multimap"] } diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index 926e5b8c..825af983 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -46,33 +46,50 @@ impl Generator { fn gen_space_file_get_primary_index_info_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let literal_name = name_generator.get_work_table_literal_name(); + let row_ident = name_generator.get_row_type_ident(); + let inner_size_ident = name_generator.get_page_inner_size_const_ident(); quote! { fn get_primary_index_info(&self) -> eyre::Result>> { + + let data: Vec::WrappedRow,#inner_size_ident >>> = self.data.clone() + .into_iter() + .enumerate() + .map(|(i, page)| { + let mut d = Data::from_data_page(page); + d.set_page_id(PageId((i + 1) as u32)); + std::sync::Arc::new(d) + }) + .collect(); + + let data_pages = DataPages::<#row_ident, #inner_size_ident>::from_data(data); + + + let mut info = { let inner = SpaceInfoPage { - id: 0.into(), - page_count: 0, - name: #literal_name.to_string(), - pk_gen_state: (), - empty_links_list: vec![], - primary_key_fields: vec![], - row_schema: vec![], - secondary_index_types: vec![], - }; - let header = GeneralHeader { - data_version: DATA_VERSION, - page_id: 0.into(), - previous_id: 0.into(), - next_id: 0.into(), - page_type: PageType::SpaceInfo, - space_id: 0.into(), - data_length: 0, - }; - GeneralPage { - header, - inner - } + id: 0.into(), + page_count: 0, + name: #literal_name.to_string(), + pk_gen_state: (), + empty_links_list: data_pages.get_empty_links(), + primary_key_fields: #row_ident::primary_key_fields(), + row_schema: #row_ident::row_schema(), + secondary_index_types: vec![], + }; + let header = GeneralHeader { + data_version: DATA_VERSION, + page_id: 0.into(), + previous_id: 0.into(), + next_id: 0.into(), + page_type: PageType::SpaceInfo, + space_id: 0.into(), + data_length: 0, + }; + GeneralPage { + header, + inner + } }; info.inner.page_count = self.primary_index.0.len() as u32 + self.primary_index.1.len() as u32; Ok(info) diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index b9874ba6..f69b6c98 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -14,7 +14,7 @@ impl Generator { let from_file_fn = self.gen_worktable_from_file_fn(); let wait_for_ops_fn = self.gen_worktable_wait_for_ops_fn(); - quote! { + let t = quote! { impl #ident { #space_info_fn #persisted_pk_fn @@ -23,7 +23,9 @@ impl Generator { #from_file_fn #wait_for_ops_fn } - } + }; + println!("YYY{} ", t); + t } fn gen_worktable_wait_for_ops_fn(&self) -> TokenStream { @@ -66,6 +68,7 @@ impl Generator { fn gen_worktable_space_info_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident); let pk = name_generator.get_primary_key_type_ident(); + let row_ident = name_generator.get_row_type_ident(); let literal_name = name_generator.get_work_table_literal_name(); quote! { @@ -76,8 +79,8 @@ impl Generator { name: #literal_name.to_string(), pk_gen_state: <<#pk as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State::default(), empty_links_list: vec![], - primary_key_fields: vec![], - row_schema: vec![], + primary_key_fields: #row_ident::primary_key_fields(), + row_schema: #row_ident::row_schema(), secondary_index_types: vec![], }; let header = GeneralHeader { diff --git a/codegen/src/worktable/generator/row.rs b/codegen/src/worktable/generator/row.rs index db3b779c..96ec031a 100644 --- a/codegen/src/worktable/generator/row.rs +++ b/codegen/src/worktable/generator/row.rs @@ -28,7 +28,8 @@ impl Generator { .pk .clone() .expect("should be set in `Generator` at this point"); - let primary_key_columns_clone = if primary_key.values.len() == 1 { + + let primary_key_columns = if primary_key.values.len() == 1 { let pk_field = primary_key .values .keys() @@ -52,11 +53,33 @@ impl Generator { } }; + let row_schema_entries = self.columns.columns_map.iter().map(|(k, v)| { + let name_str = k.to_string(); + let type_str = v.to_string(); + quote! { + (#name_str.to_string(), #type_str.to_string()) + } + }); + + let primary_key_fields = primary_key.values.keys().map(|key| { + let key_str = key.to_string(); + quote! { + #key_str.to_string() + } + }); + quote! { impl TableRow<#primary_key_ident> for #ident { - fn get_primary_key(&self) -> #primary_key_ident { - #primary_key_columns_clone + #primary_key_columns + } + + fn row_schema() -> Vec<(String, String)> { + vec![#(#row_schema_entries),*] + } + + fn primary_key_fields() -> Vec { + vec![#(#primary_key_fields),*] } } } diff --git a/examples/src/main.rs b/examples/src/main.rs index 63b95647..aeefbb0e 100644 --- a/examples/src/main.rs +++ b/examples/src/main.rs @@ -50,14 +50,15 @@ async fn main() { attr_float: 100.0.into(), attr_string: "String_attr0".to_string(), }; + let pk: MyPrimaryKey = my_table.insert(row).expect("primary key"); - for i in 2..1000000_i64 { + for i in 2..10 { let row = MyRow { val: 777, attr: format!("Attribute{}", i), attr2: 345 + i as i32, test: i as i32, - id: i as u64, + id: (i - 1) as u64, attr_float: (100.0 + i as f64).into(), attr_string: format!("String_attr{}", i), }; @@ -66,7 +67,6 @@ async fn main() { } // insert - let pk: MyPrimaryKey = my_table.insert(row).expect("primary key"); // Select ALL records from WT let _select_all = my_table.select_all().execute(); @@ -87,8 +87,8 @@ async fn main() { //} // Update Value query - let update = my_table.update_val_by_id(ValByIdQuery { val: 1337 }, pk.clone()); - let _ = block_on(update); + // let update = my_table.update_val_by_id(ValByIdQuery { val: 1337 }, pk.clone()); + //let _ = block_on(update); let _select_all = my_table.select_all().execute(); //println!("Select after update val {:?}", select_all); @@ -102,4 +102,6 @@ async fn main() { let info = my_table.system_info(); println!("{info}"); + // my_table.wait_for_ops().await; + let _ = my_table.persist().await.unwrap(); } diff --git a/src/lib.rs b/src/lib.rs index 75f7ed16..346c7391 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,7 +41,7 @@ pub mod prelude { pub use data_bucket::{ align, get_index_page_size_from_data_length, map_data_pages_to_general, parse_data_page, parse_page, persist_page, seek_to_page_start, update_at, DataPage, GeneralHeader, - GeneralPage, IndexPage, Interval, Link, PageType, Persistable, PersistableIndex, + GeneralPage, IndexPage, Interval, Link, PageId, PageType, Persistable, PersistableIndex, SizeMeasurable, SizeMeasure, SpaceInfoPage, TableOfContentsPage, DATA_VERSION, GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, PAGE_SIZE, }; diff --git a/src/row.rs b/src/row.rs index 7b005af3..988d78da 100644 --- a/src/row.rs +++ b/src/row.rs @@ -1,3 +1,5 @@ pub trait TableRow { fn get_primary_key(&self) -> Pk; + fn row_schema() -> Vec<(String, String)>; + fn primary_key_fields() -> Vec; }