WIP Cli tool for read WT files

Cargo.toml

@@ -25,8 +25,8 @@ lockfree = { version = "0.5.1" }
 worktable_codegen = { path = "codegen", version = "0.5.1" }
 futures = "0.3.30"
 uuid = { version = "1.10.0", features = ["v4"] }
-data_bucket = { git = "https://github.com/pathscale/DataBucket", rev = "3eb4fc2" }
-#data_bucket = { path = "../DataBucket", version = "0.2.1" }
+#data_bucket = { git = "https://github.com/pathscale/DataBucket", rev = "3eb4fc2" }
+data_bucket = { path = "../DataBucket", version = "0.2.1" }
 performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true }
 performance_measurement = { path = "performance_measurement", version = "0.1.0", optional = true }
 indexset = { version = "0.11.3", features = ["concurrent", "cdc", "multimap"] }

codegen/src/persist_table/generator/space_file/mod.rs

@@ -46,33 +46,50 @@ impl Generator {
     fn gen_space_file_get_primary_index_info_fn(&self) -> TokenStream {
         let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident);
         let literal_name = name_generator.get_work_table_literal_name();
+        let row_ident = name_generator.get_row_type_ident();
+        let inner_size_ident = name_generator.get_page_inner_size_const_ident();
 
         quote! {
             fn get_primary_index_info(&self) -> eyre::Result<GeneralPage<SpaceInfoPage<()>>> {
+
+                let data: Vec<std::sync::Arc<Data<<#row_ident as StorableRow>::WrappedRow,#inner_size_ident >>> = self.data.clone()
+                     .into_iter()
+                     .enumerate()
+                     .map(|(i, page)| {
+                         let mut d = Data::from_data_page(page);
+                         d.set_page_id(PageId((i + 1) as u32));
+                         std::sync::Arc::new(d)
+                     })
+                     .collect();
+
+                let data_pages = DataPages::<#row_ident, #inner_size_ident>::from_data(data);
+
+
+
                 let mut info = {
                     let inner = SpaceInfoPage {
-                    id: 0.into(),
-                    page_count: 0,
-                    name: #literal_name.to_string(),
-                    pk_gen_state: (),
-                    empty_links_list: vec![],
-                    primary_key_fields: vec![],
-                    row_schema: vec![],
-                    secondary_index_types: vec![],
-                };
-                let header = GeneralHeader {
-                    data_version: DATA_VERSION,
-                    page_id: 0.into(),
-                    previous_id: 0.into(),
-                    next_id: 0.into(),
-                    page_type: PageType::SpaceInfo,
-                    space_id: 0.into(),
-                    data_length: 0,
-                };
-                GeneralPage {
-                    header,
-                    inner
-                }
+                        id: 0.into(),
+                        page_count: 0,
+                        name: #literal_name.to_string(),
+                        pk_gen_state: (),
+                        empty_links_list: data_pages.get_empty_links(),
+                        primary_key_fields: #row_ident::primary_key_fields(),
+                        row_schema: #row_ident::row_schema(),
+                        secondary_index_types: vec![],
+                    };
+                    let header = GeneralHeader {
+                        data_version: DATA_VERSION,
+                        page_id: 0.into(),
+                        previous_id: 0.into(),
+                        next_id: 0.into(),
+                        page_type: PageType::SpaceInfo,
+                        space_id: 0.into(),
+                        data_length: 0,
+                    };
+                    GeneralPage {
+                        header,
+                        inner
+                    }
                 };
                 info.inner.page_count = self.primary_index.0.len() as u32 + self.primary_index.1.len() as u32;
                 Ok(info)

codegen/src/persist_table/generator/space_file/worktable_impls.rs

@@ -14,7 +14,7 @@ impl Generator {
         let from_file_fn = self.gen_worktable_from_file_fn();
         let wait_for_ops_fn = self.gen_worktable_wait_for_ops_fn();
 
-        quote! {
+        let t = quote! {
             impl #ident {
                 #space_info_fn
                 #persisted_pk_fn
@@ -23,7 +23,9 @@ impl Generator {
                 #from_file_fn
                 #wait_for_ops_fn
             }
-        }
+        };
+        println!("YYY{} ", t);
+        t
     }
 
     fn gen_worktable_wait_for_ops_fn(&self) -> TokenStream {
@@ -66,6 +68,7 @@ impl Generator {
     fn gen_worktable_space_info_fn(&self) -> TokenStream {
         let name_generator = WorktableNameGenerator::from_struct_ident(&self.struct_def.ident);
         let pk = name_generator.get_primary_key_type_ident();
+        let row_ident = name_generator.get_row_type_ident();
         let literal_name = name_generator.get_work_table_literal_name();
 
         quote! {
@@ -76,8 +79,8 @@ impl Generator {
                     name: #literal_name.to_string(),
                     pk_gen_state: <<#pk as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State::default(),
                     empty_links_list: vec![],
-                    primary_key_fields: vec![],
-                    row_schema: vec![],
+                    primary_key_fields: #row_ident::primary_key_fields(),
+                    row_schema: #row_ident::row_schema(),
                     secondary_index_types: vec![],
                 };
                 let header = GeneralHeader {

codegen/src/worktable/generator/row.rs

@@ -28,7 +28,8 @@ impl Generator {
             .pk
             .clone()
             .expect("should be set in `Generator` at this point");
-        let primary_key_columns_clone = if primary_key.values.len() == 1 {
+
+        let primary_key_columns = if primary_key.values.len() == 1 {
             let pk_field = primary_key
                 .values
                 .keys()
@@ -52,11 +53,33 @@ impl Generator {
             }
         };
 
+        let row_schema_entries = self.columns.columns_map.iter().map(|(k, v)| {
+            let name_str = k.to_string();
+            let type_str = v.to_string();
+            quote! {
+                (#name_str.to_string(), #type_str.to_string())
+            }
+        });
+
+        let primary_key_fields = primary_key.values.keys().map(|key| {
+            let key_str = key.to_string();
+            quote! {
+                #key_str.to_string()
+            }
+        });
+
         quote! {
             impl TableRow<#primary_key_ident> for #ident {
-
                 fn get_primary_key(&self) -> #primary_key_ident {
-                    #primary_key_columns_clone
+                    #primary_key_columns
+                }
+
+                fn row_schema() -> Vec<(String, String)> {
+                    vec![#(#row_schema_entries),*]
+                }
+
+                fn primary_key_fields() -> Vec<String> {
+                    vec![#(#primary_key_fields),*]
                 }
             }
         }

examples/src/main.rs

@@ -50,14 +50,15 @@ async fn main() {
         attr_float: 100.0.into(),
         attr_string: "String_attr0".to_string(),
     };
+    let pk: MyPrimaryKey = my_table.insert(row).expect("primary key");
 
-    for i in 2..1000000_i64 {
+    for i in 2..10 {
         let row = MyRow {
             val: 777,
             attr: format!("Attribute{}", i),
             attr2: 345 + i as i32,
             test: i as i32,
-            id: i as u64,
+            id: (i - 1) as u64,
             attr_float: (100.0 + i as f64).into(),
             attr_string: format!("String_attr{}", i),
         };
@@ -66,7 +67,6 @@ async fn main() {
     }
 
     // insert
-    let pk: MyPrimaryKey = my_table.insert(row).expect("primary key");
 
     // Select ALL records from WT
     let _select_all = my_table.select_all().execute();
@@ -87,8 +87,8 @@ async fn main() {
     //}
 
     // Update Value query
-    let update = my_table.update_val_by_id(ValByIdQuery { val: 1337 }, pk.clone());
-    let _ = block_on(update);
+    //  let update = my_table.update_val_by_id(ValByIdQuery { val: 1337 }, pk.clone());
+    //let _ = block_on(update);
 
     let _select_all = my_table.select_all().execute();
     //println!("Select after update val {:?}", select_all);
@@ -102,4 +102,6 @@ async fn main() {
     let info = my_table.system_info();
 
     println!("{info}");
+    //    my_table.wait_for_ops().await;
+    let _ = my_table.persist().await.unwrap();
 }

src/lib.rs

@@ -41,7 +41,7 @@ pub mod prelude {
     pub use data_bucket::{
         align, get_index_page_size_from_data_length, map_data_pages_to_general, parse_data_page,
         parse_page, persist_page, seek_to_page_start, update_at, DataPage, GeneralHeader,
-        GeneralPage, IndexPage, Interval, Link, PageType, Persistable, PersistableIndex,
+        GeneralPage, IndexPage, Interval, Link, PageId, PageType, Persistable, PersistableIndex,
         SizeMeasurable, SizeMeasure, SpaceInfoPage, TableOfContentsPage, DATA_VERSION,
         GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, PAGE_SIZE,
     };

src/row.rs

@@ -1,3 +1,5 @@
 pub trait TableRow<Pk> {
     fn get_primary_key(&self) -> Pk;
+    fn row_schema() -> Vec<(String, String)>;
+    fn primary_key_fields() -> Vec<String>;
 }