Optimizing raw transactions format (for saving storage) (#70)

spacemeshos · Feb 18, 2020 · a34aa46 · a34aa46
1 parent 990ad3f
commit a34aa46
Show file tree

Hide file tree

Showing 58 changed files with 3,028 additions and 2,760 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -29,13 +29,12 @@ features = ["lz4"]
 default-features = false
 git = "https://github.com/spacemeshos/wasmer"
 branch = "develop"
-features = ["default-backend-singlepass"]
+features = ["singlepass"]
 
 [dependencies.wasmer-runtime-core]
 default-features = false
 git = "https://github.com/spacemeshos/wasmer"
 branch = "develop"
-features = ["backend-singlepass"]
 
 [dependencies.wasmer-runtime-c-api]
 default-features = false
@@ -71,8 +70,8 @@ members = [
   "crates/svm-common",
   "crates/svm-kv",
   "crates/svm-storage",
-  "crates/svm-compiler",
   "crates/svm-runtime",
+  "crates/svm-compiler",
   "crates/svm-runtime-c-api",
 ]
 

diff --git a/crates/svm-app/Cargo.toml b/crates/svm-app/Cargo.toml
@@ -25,6 +25,9 @@ version = "1.3.2"
 [dependencies.log]
 version = "0.4"
 
+[dependencies.bit-vec]
+version = "0.6.1"
+
 [features]
 default = ["memory", "default-rocksdb"]
 memory = ["svm-kv/memory"]

diff --git a/crates/svm-app/src/error/parse.rs b/crates/svm-app/src/error/parse.rs
@@ -5,33 +5,35 @@ use crate::raw::Field;
 #[allow(missing_docs)]
 #[derive(PartialEq, Clone)]
 pub enum ParseError {
-    InvalidWasm,
-    InvalidArgType(u8),
+    ExpectedEOF,
     EmptyField(Field),
-    EmptyList(Field),
     NotEnoughBytes(Field),
+    TooManyBytes(Field),
     NotSupported(Field),
     InvalidUTF8String(Field),
     InvalidProtocolVersion(u32),
+    FuncArgValueIncomplete {
+        arg_idx: usize,
+        expected_nibbles: usize,
+        actual_read: usize,
+    },
 }
 
 impl fmt::Display for ParseError {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
         match self {
+            ParseError::ExpectedEOF => write!(fmt, "Expected EOF but there are more left bytes"),
             ParseError::EmptyField(f) => write!(fmt, "Field `{}` must not be empty", f),
             ParseError::NotEnoughBytes(f) => write!(fmt, "Not enough bytes for field `{}`", f),
-            ParseError::InvalidProtocolVersion(v) => {
-                write!(fmt, "Unsupported protocol version: `{}`", v)
-            }
+            ParseError::TooManyBytes(f) => write!(fmt, "Too many bytes for field `{}`", f),
+            ParseError::InvalidProtocolVersion(msg) => write!(fmt, "{}", msg),
             ParseError::NotSupported(f) => write!(fmt, "Feature `{}` is not supported yet", f),
-            ParseError::EmptyList(f) => {
-                write!(fmt, "`{}`-(s) list must contain at least one item", f)
-            }
             ParseError::InvalidUTF8String(f) => {
                 write!(fmt, "Field `{}` must be a valid UTF-8 string", f)
             }
-            ParseError::InvalidWasm => write!(fmt, "Invalid wasm format"),
-            ParseError::InvalidArgType(..) => write!(fmt, "Invalid arg type"),
+            ParseError::FuncArgValueIncomplete { .. } => {
+                write!(fmt, "Function argument is incomplete (missing data)")
+            }
         }
     }
 }

diff --git a/crates/svm-app/src/raw/app/mod.rs b/crates/svm-app/src/raw/app/mod.rs
@@ -1,19 +1,15 @@
-//!          `Spawn-App` Raw Format Version 0.0.0.0
+//!                   `Spawn-App` Raw Format
 //!  ----------------------------------------------------------
-//!  |     proto     |                                        |
-//!  |    version    |        `AppTemplate` `Address`         |
-//!  |    (4 bytes)  |            (20 bytes)                  |
+//!  |               |                                        |
+//!  |    `version`  |        `AppTemplate` (`Address`)       |
 //!  |_______________|________________________________________|
-//!  |  ctor-buf  |  ctor-buf  |  ctor-buf  |                 |
-//!  |   #slices  |  slice #1  |  slice #1  |                 |
-//!  |            |   len      |            |     . . . .     |
-//!  | (1 byte)   | (2 bytes)  |   value    |                 |
-//!  |____________|____________|____________|_________________|
-//!  | ctor-func | ctor-func | ctor-func |                    |
-//!  |  #args    |  arg #1   |   arg #1  |                    |
-//!  |           |   type    |   value   |      . . . .       |
-//!  | (1 byte)  | (1 byte)  | (u32/u64) |                    |
-//!  |___________|___________|___________|____________________|
+//!  |            |                                           |
+//!  | ctor index |            ctor-buf (`func-buf`)          |
+//!  |____________|___________________________________________|
+//!  |                                                        |
+//!  |                 ctor-args (`func-args``)               |
+//!  |________________________________________________________|
+//!
 //!
 
 mod parse;

diff --git a/crates/svm-app/src/raw/app/parse.rs b/crates/svm-app/src/raw/app/parse.rs
@@ -1,9 +1,7 @@
-use std::io::Cursor;
-
 use crate::{
     error::ParseError,
-    raw::{helpers, Field},
-    types::{App, BufferSlice, SpawnApp},
+    raw::{helpers, Field, NibbleIter},
+    types::{App, SpawnApp},
 };
 
 use svm_common::Address;
@@ -13,13 +11,16 @@ use svm_common::Address;
 /// On failure, returns `ParseError`
 #[must_use]
 pub fn parse_app(bytes: &[u8], creator: &Address) -> Result<SpawnApp, ParseError> {
-    let mut cursor = Cursor::new(bytes);
+    let mut iter = NibbleIter::new(bytes);
+
+    helpers::decode_version(&mut iter)?;
 
-    helpers::parse_version(&mut cursor)?;
+    let template = helpers::decode_address(&mut iter, Field::AppTemplate)?;
+    let ctor_idx = decode_ctor_index(&mut iter)?;
+    let ctor_buf = helpers::decode_func_buf(&mut iter)?;
+    let ctor_args = helpers::decode_func_args(&mut iter)?;
 
-    let template = helpers::parse_address(&mut cursor, Field::AppTemplate)?;
-    let ctor_buf = helpers::parse_func_buf(&mut cursor)?;
-    let ctor_args = helpers::parse_func_args(&mut cursor)?;
+    helpers::ensure_eof(&mut iter);
 
     let app = App {
         template,
@@ -28,9 +29,15 @@ pub fn parse_app(bytes: &[u8], creator: &Address) -> Result<SpawnApp, ParseError
 
     let spawn_app = SpawnApp {
         app,
+        ctor_idx,
         ctor_buf,
         ctor_args,
     };
 
     Ok(spawn_app)
 }
+
+#[must_use]
+fn decode_ctor_index(iter: &mut NibbleIter) -> Result<u16, ParseError> {
+    helpers::decode_varuint14(iter, Field::FuncIndex)
+}
diff --git a/crates/svm-app/src/raw/field.rs b/crates/svm-app/src/raw/field.rs
@@ -1,28 +1,19 @@
 use std::fmt;
 
 #[allow(missing_docs)]
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, PartialEq, Copy, Clone)]
 pub enum Field {
     Version,
     NameLength,
     Name,
-    Admins,
-    AdminsCount,
-    CodeLength,
-    Dependencies,
-    DependenciesCount,
     PageCount,
     Code,
     AppTemplate,
     App,
-    FuncNameLength,
-    FuncName,
-    FuncBufSlicesCount,
-    FuncBufSliceLength,
-    FuncBufSlice,
-    FuncArgsCount,
-    WasmType,
-    WasmValue,
+    FuncIndex,
+    FuncBufLength,
+    FuncBuf,
+    FuncArgsNoMoreMark,
 }
 
 impl fmt::Display for Field {