Skip to content

Commit

Permalink
Update snapshot message format
Browse files Browse the repository at this point in the history
  • Loading branch information
@remagpie @foriequal0
remagpie authored and foriequal0 committed Dec 10, 2019
1 parent f4af598 commit be35914
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 158 deletions.
50 changes: 10 additions & 40 deletions spec/Block-Synchronization-Extension.md
View file
Expand Up @@ -53,32 +53,17 @@ Request corresponding bodies for each hash.
* Restriction:
* MUST include at least one item


### GetStateHead

```
GetStateHead(block_hash)
```

Request corresponding state head for block of `block_hash`.

* Identifier: 0x06
* Restriction: Block number of requested block MUST be multiple of 214.


### GetStateChunk

```
GetStateChunk(block_hash, tree_root)
GetStateChunk(block_hash, [...chunk_roots])
```

Request entire subtree starting from `tree_root`.
Request corresponding snapshot chunk for each `chunk_root`.

* Identifier: 0x08
* Identifier: 0x0a
* Restriction:
* Block number of requested block MUST be multiple of 214.
* `tree_root` MUST be included in requested block’s state trie.
* Depth of `tree_root` inside state trie MUST be equal to 2. (Depth of state root is 0)
* All values in `[...chunk_roots]` MUST be included in requested block’s state trie.


## Response messages
Expand Down Expand Up @@ -113,30 +98,15 @@ Response to `GetBodies` message. Snappy algorithm is used to compress content.
* If received body is zero-length array, it means either body value is [], or sender doesn’t have body for requested hash


### StateHead

```
StateHead(compressed((key_0, value_0), …) | [])
```

Response to `GetStateHead` message. Key and value included in this messages are raw value stored in state trie. Snappy algorithm is used for compression of content.

* Identifier: 0x07
* Restriction:
* State root of requested block MUST be included
* For all nodes with depth of less than 2 included in this message, all of its child MUST also be included.
* Content MUST be empty array if sender didn’t have requested data


### StateChunk
```
StateChunk(compressed((key_0, value_0), …) | [])
StateChunk([compressed([terminal_0, …] | []), ...])
```

Response to `GetStateChunk` message. Details of message is same as `StateHead` message.
Response to `GetStateChunk` message. Snappy algorithm is used for compression of content.

* Identifier: 0x09
* Identifier: 0x0b
* Restriction:
* Node corresponding to tree_root in request MUST be included
* Every nodes included in message MUST have all of its child in same message.
* Content MUST be empty array if sender didn’t have requested data
* Number and order of chunks included in this message MUST be equal to request information.
* Node corresponding to `chunk_root` in request MUST be included
* If sender doesn’t have a chunk for the requested hash, corresponding chunk MUST be compressed([]), not omitted.
33 changes: 6 additions & 27 deletions sync/src/block/extension.rs
View file
Expand Up @@ -49,8 +49,6 @@ const SYNC_EXPIRE_TOKEN_END: TimerToken = SYNC_EXPIRE_TOKEN_BEGIN + SYNC_EXPIRE_
const SYNC_TIMER_INTERVAL: u64 = 1000;
const SYNC_EXPIRE_REQUEST_INTERVAL: u64 = 15000;

const SNAPSHOT_PERIOD: u64 = (1 << 14);

#[derive(Debug, PartialEq)]
pub struct TokenInfo {
node_id: NodeId,
Expand Down Expand Up @@ -563,11 +561,9 @@ impl Extension {
ctrace!(SYNC, "Received body request from {}", from);
self.create_bodies_response(hashes)
}
RequestMessage::StateHead(hash) => self.create_state_head_response(hash),
RequestMessage::StateChunk {
block_hash,
tree_root,
} => self.create_state_chunk_response(block_hash, tree_root),
RequestMessage::StateChunk(block_hash, chunk_root) => {
self.create_state_chunk_response(block_hash, chunk_root)
}
};

self.api.send(from, Arc::new(Message::Response(id, response).rlp_bytes()));
Expand All @@ -579,21 +575,9 @@ impl Extension {
..
} => true,
RequestMessage::Bodies(hashes) => !hashes.is_empty(),
RequestMessage::StateHead(hash) => match self.client.block_number(&BlockId::Hash(*hash)) {
Some(number) if number % SNAPSHOT_PERIOD == 0 => true,
_ => false,
},
RequestMessage::StateChunk {
block_hash,
..
} => {
let _is_checkpoint = match self.client.block_number(&BlockId::Hash(*block_hash)) {
Some(number) if number % SNAPSHOT_PERIOD == 0 => true,
_ => false,
};
// FIXME: check tree_root
unimplemented!()
}
} => unimplemented!(),
}
}

Expand Down Expand Up @@ -631,11 +615,7 @@ impl Extension {
ResponseMessage::Bodies(bodies)
}

fn create_state_head_response(&self, _hash: BlockHash) -> ResponseMessage {
unimplemented!()
}

fn create_state_chunk_response(&self, _hash: BlockHash, _tree_root: H256) -> ResponseMessage {
fn create_state_chunk_response(&self, _hash: BlockHash, _tree_root: Vec<H256>) -> ResponseMessage {
unimplemented!()
}

Expand Down Expand Up @@ -676,7 +656,7 @@ impl Extension {
self.on_body_response(hashes, bodies);
self.check_sync_variable();
}
_ => unimplemented!(),
ResponseMessage::StateChunk(..) => unimplemented!(),
}
}
}
Expand Down Expand Up @@ -730,7 +710,6 @@ impl Extension {
}
true
}
(RequestMessage::StateHead(..), ResponseMessage::StateHead(..)) => unimplemented!(),
(
RequestMessage::StateChunk {
..
Expand Down
21 changes: 6 additions & 15 deletions sync/src/block/message/mod.rs
View file
Expand Up @@ -29,10 +29,8 @@ const MESSAGE_ID_GET_HEADERS: u8 = 0x02;
const MESSAGE_ID_HEADERS: u8 = 0x03;
const MESSAGE_ID_GET_BODIES: u8 = 0x04;
const MESSAGE_ID_BODIES: u8 = 0x05;
const MESSAGE_ID_GET_STATE_HEAD: u8 = 0x06;
const MESSAGE_ID_STATE_HEAD: u8 = 0x07;
const MESSAGE_ID_GET_STATE_CHUNK: u8 = 0x08;
const MESSAGE_ID_STATE_CHUNK: u8 = 0x09;
const MESSAGE_ID_GET_STATE_CHUNK: u8 = 0x0a;
const MESSAGE_ID_STATE_CHUNK: u8 = 0x0b;

#[derive(Debug, PartialEq)]
pub enum Message {
Expand Down Expand Up @@ -114,11 +112,10 @@ impl Decodable for Message {
let request_id = rlp.val_at(1)?;
let message = rlp.at(2)?;
match id {
MESSAGE_ID_GET_HEADERS
| MESSAGE_ID_GET_BODIES
| MESSAGE_ID_GET_STATE_HEAD
| MESSAGE_ID_GET_STATE_CHUNK => Ok(Message::Request(request_id, RequestMessage::decode(id, &message)?)),
MESSAGE_ID_HEADERS | MESSAGE_ID_BODIES | MESSAGE_ID_STATE_HEAD | MESSAGE_ID_STATE_CHUNK => {
MESSAGE_ID_GET_HEADERS | MESSAGE_ID_GET_BODIES | MESSAGE_ID_GET_STATE_CHUNK => {
Ok(Message::Request(request_id, RequestMessage::decode(id, &message)?))
}
MESSAGE_ID_HEADERS | MESSAGE_ID_BODIES | MESSAGE_ID_STATE_CHUNK => {
Ok(Message::Response(request_id, ResponseMessage::decode(id, &message)?))
}
_ => Err(DecoderError::Custom("Unknown message id detected")),
Expand Down Expand Up @@ -148,10 +145,4 @@ mod tests {
let request_id = 10;
rlp_encode_and_decode_test!(Message::Request(request_id, RequestMessage::Bodies(vec![])));
}

#[test]
fn request_state_head_rlp() {
let request_id = 10;
rlp_encode_and_decode_test!(Message::Request(request_id, RequestMessage::StateHead(H256::random().into())));
}
}
44 changes: 5 additions & 39 deletions sync/src/block/message/request.rs
View file
Expand Up @@ -25,11 +25,7 @@ pub enum RequestMessage {
max_count: u64,
},
Bodies(Vec<BlockHash>),
StateHead(BlockHash),
StateChunk {
block_hash: BlockHash,
tree_root: H256,
},
StateChunk(BlockHash, Vec<H256>),
}

impl Encodable for RequestMessage {
Expand All @@ -46,17 +42,10 @@ impl Encodable for RequestMessage {
RequestMessage::Bodies(hashes) => {
s.append_list(hashes);
}
RequestMessage::StateHead(block_hash) => {
s.begin_list(1);
s.append(block_hash);
}
RequestMessage::StateChunk {
block_hash,
tree_root,
} => {
RequestMessage::StateChunk(block_hash, merkle_roots) => {
s.begin_list(2);
s.append(block_hash);
s.append(tree_root);
s.append_list(merkle_roots);
}
};
}
Expand All @@ -69,7 +58,6 @@ impl RequestMessage {
..
} => super::MESSAGE_ID_GET_HEADERS,
RequestMessage::Bodies(..) => super::MESSAGE_ID_GET_BODIES,
RequestMessage::StateHead(..) => super::MESSAGE_ID_GET_STATE_HEAD,
RequestMessage::StateChunk {
..
} => super::MESSAGE_ID_GET_STATE_CHUNK,
Expand All @@ -92,16 +80,6 @@ impl RequestMessage {
}
}
super::MESSAGE_ID_GET_BODIES => RequestMessage::Bodies(rlp.as_list()?),
super::MESSAGE_ID_GET_STATE_HEAD => {
let item_count = rlp.item_count()?;
if item_count != 1 {
return Err(DecoderError::RlpIncorrectListLen {
got: item_count,
expected: 1,
})
}
RequestMessage::StateHead(rlp.val_at(0)?)
}
super::MESSAGE_ID_GET_STATE_CHUNK => {
let item_count = rlp.item_count()?;
if item_count != 2 {
Expand All @@ -110,10 +88,7 @@ impl RequestMessage {
expected: 2,
})
}
RequestMessage::StateChunk {
block_hash: rlp.val_at(0)?,
tree_root: rlp.val_at(1)?,
}
RequestMessage::StateChunk(rlp.val_at(0)?, rlp.list_at(1)?)
}
_ => return Err(DecoderError::Custom("Unknown message id detected")),
};
Expand Down Expand Up @@ -149,18 +124,9 @@ mod tests {
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}

#[test]
fn request_state_head_message_rlp() {
let message = RequestMessage::StateHead(H256::default().into());
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}

#[test]
fn request_state_chunk_message_rlp() {
let message = RequestMessage::StateChunk {
block_hash: H256::default().into(),
tree_root: H256::default(),
};
let message = RequestMessage::StateChunk(H256::default().into(), vec![H256::default()]);
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}
}
40 changes: 4 additions & 36 deletions sync/src/block/message/response.rs
View file
Expand Up @@ -24,8 +24,7 @@ use ctypes::Header;
pub enum ResponseMessage {
Headers(Vec<Header>),
Bodies(Vec<Vec<UnverifiedTransaction>>),
StateHead(Vec<u8>),
StateChunk(Vec<u8>),
StateChunk(Vec<Vec<u8>>),
}

impl Encodable for ResponseMessage {
Expand Down Expand Up @@ -53,13 +52,8 @@ impl Encodable for ResponseMessage {

s.append(&compressed);
}
ResponseMessage::StateHead(bytes) => {
s.begin_list(1);
s.append(bytes);
}
ResponseMessage::StateChunk(bytes) => {
s.begin_list(1);
s.append(bytes);
ResponseMessage::StateChunk(chunks) => {
s.append_list::<Vec<u8>, Vec<u8>>(chunks);
}
};
}
Expand All @@ -72,7 +66,6 @@ impl ResponseMessage {
..
} => super::MESSAGE_ID_HEADERS,
ResponseMessage::Bodies(..) => super::MESSAGE_ID_BODIES,
ResponseMessage::StateHead(..) => super::MESSAGE_ID_STATE_HEAD,
ResponseMessage::StateChunk {
..
} => super::MESSAGE_ID_STATE_CHUNK,
Expand Down Expand Up @@ -109,26 +102,7 @@ impl ResponseMessage {
}
ResponseMessage::Bodies(bodies)
}
super::MESSAGE_ID_STATE_HEAD => {
let item_count = rlp.item_count()?;
if item_count != 1 {
return Err(DecoderError::RlpIncorrectListLen {
got: item_count,
expected: 1,
})
}
ResponseMessage::StateHead(rlp.val_at(0)?)
}
super::MESSAGE_ID_STATE_CHUNK => {
let item_count = rlp.item_count()?;
if item_count != 1 {
return Err(DecoderError::RlpIncorrectListLen {
got: item_count,
expected: 1,
})
}
ResponseMessage::StateChunk(rlp.val_at(0)?)
}
super::MESSAGE_ID_STATE_CHUNK => ResponseMessage::StateChunk(rlp.as_list()?),
_ => return Err(DecoderError::Custom("Unknown message id detected")),
};

Expand Down Expand Up @@ -184,12 +158,6 @@ mod tests {
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}

#[test]
fn state_head_message_rlp() {
let message = ResponseMessage::StateHead(vec![]);
assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
}

#[test]
fn state_chunk_message_rlp() {
let message = ResponseMessage::StateChunk(vec![]);
Expand Down
2 changes: 1 addition & 1 deletion util/merkle/src/snapshot/mod.rs
View file
Expand Up @@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

mod chunk;
pub mod chunk;
mod compress;
mod error;
mod ordered_heap;
Expand Down

0 comments on commit be35914

Please sign in to comment.