From 39f8cfffc67ff75be75081d286a854d9d4aa6d6c Mon Sep 17 00:00:00 2001 From: patrick Date: Sat, 5 Nov 2022 15:08:42 +0000 Subject: [PATCH 01/13] replace get_object_owned_by_object with dynamic field added dynamic_field index --- crates/sui-core/src/authority.rs | 13 +- .../sui-core/src/authority/authority_store.rs | 290 +++++++++++++++--- .../src/authority/authority_store_tables.rs | 4 +- .../src/unit_tests/authority_tests.rs | 12 +- crates/sui-json-rpc-types/src/lib.rs | 4 + crates/sui-json-rpc/src/api.rs | 6 +- crates/sui-json-rpc/src/read_api.rs | 58 ++-- .../src/transaction_builder_api.rs | 3 +- crates/sui-rosetta/src/account.rs | 3 +- crates/sui-rosetta/src/construction.rs | 3 +- crates/sui-sdk/src/lib.rs | 8 +- crates/sui-types/src/base_types.rs | 21 +- crates/sui/src/client_commands.rs | 6 - 13 files changed, 337 insertions(+), 94 deletions(-) diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index 13b8ba737f95f..71169ab863e7d 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -58,7 +58,7 @@ use sui_types::committee::EpochId; use sui_types::crypto::{AuthorityKeyPair, NetworkKeyPair}; use sui_types::event::{Event, EventID}; use sui_types::messages_checkpoint::{CheckpointRequest, CheckpointResponse}; -use sui_types::object::{Owner, PastObjectRead}; +use sui_types::object::PastObjectRead; use sui_types::query::{EventQuery, TransactionQuery}; use sui_types::sui_system_state::SuiSystemState; use sui_types::temporary_store::InnerTemporaryStore; @@ -1861,15 +1861,12 @@ impl AuthorityState { } } - pub fn get_owner_objects(&self, owner: Owner) -> SuiResult> { + pub fn get_owner_objects(&self, owner: SuiAddress) -> SuiResult> { self.database.get_owner_objects(owner) } - pub fn get_owner_objects_iterator( - &self, - owner: Owner, - ) -> SuiResult + '_> { - self.database.get_owner_objects_iterator(owner) + pub fn get_dynamic_fields(&self, owner: ObjectID) -> SuiResult> { + self.database.get_dynamic_fields(owner) } pub fn get_total_transaction_number(&self) -> Result { @@ -2132,7 +2129,7 @@ impl AuthorityState { fn make_account_info(&self, account: SuiAddress) -> Result { self.database - .get_owner_objects(Owner::AddressOwner(account)) + .get_owner_objects(account) .map(|object_ids| AccountInfoResponse { object_ids: object_ids.into_iter().map(|id| id.into()).collect(), owner: account, diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index a5c5d1507c424..84fe1cf2eb7e7 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -1,15 +1,28 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::collections::BTreeMap; +use std::iter; +use std::path::Path; +use std::sync::Arc; +use std::{fmt::Debug, path::PathBuf}; + use super::{authority_store_tables::AuthorityPerpetualTables, *}; use crate::authority::authority_per_epoch_store::{ AuthorityPerEpochStore, ExecutionIndicesWithHash, }; use arc_swap::ArcSwap; +use fastcrypto::encoding::{Base64, Encoding}; +use itertools::Itertools; +use move_core_types::language_storage::{StructTag, TypeTag}; use once_cell::sync::OnceCell; use rocksdb::Options; use serde::{Deserialize, Serialize}; use serde_with::serde_as; +use tokio_retry::strategy::{jitter, ExponentialBackoff}; +use tracing::{debug, info, trace}; + +use narwhal_executor::ExecutionIndices; use std::collections::BTreeSet; use std::iter; use std::path::Path; @@ -24,11 +37,18 @@ use sui_types::object::Owner; use sui_types::storage::ChildObjectResolver; use sui_types::{base_types::SequenceNumber, storage::ParentSync}; use sui_types::{batch::TxSequenceNumber, object::PACKAGE_VERSION}; -use tokio_retry::strategy::{jitter, ExponentialBackoff}; -use tracing::{debug, info, trace}; use typed_store::rocks::DBBatch; use typed_store::traits::Map; +use crate::authority::authority_store_tables::ExecutionIndicesWithHash; + +use super::{ + authority_store_tables::{AuthorityEpochTables, AuthorityPerpetualTables}, + *, +}; + +pub type AuthorityStore = SuiDataStore; + const NUM_SHARDS: usize = 4096; const SHARD_SIZE: usize = 128; @@ -201,7 +221,7 @@ impl AuthorityStore { } // Methods to read the store - pub fn get_owner_objects(&self, owner: Owner) -> Result, SuiError> { + pub fn get_owner_objects(&self, owner: SuiAddress) -> Result, SuiError> { debug!(?owner, "get_owner_objects"); Ok(self.get_owner_objects_iterator(owner)?.collect()) } @@ -222,6 +242,20 @@ impl AuthorityStore { .map(|(_, object_info)| object_info)) } + // Methods to read the store + pub fn get_dynamic_fields(&self, object: ObjectID) -> Result, SuiError> { + debug!(?object, "get_dynamic_fields"); + Ok(self + .perpetual_tables + .dynamic_field_index + .iter() + // The object id 0 is the smallest possible + .skip_to(&(object, ObjectID::ZERO))? + .take_while(|((object_owner, _), _)| (object_owner == &object)) + .map(|(_, object_info)| object_info) + .collect()) + } + pub fn get_object_by_key( &self, object_id: &ObjectID, @@ -556,10 +590,22 @@ impl AuthorityStore { // Update the index if object.get_single_owner().is_some() { - self.perpetual_tables.owner_index.insert( - &(object.owner, object_ref.0), - &ObjectInfo::new(&object_ref, object), - )?; + match object.owner { + Owner::AddressOwner(addr) => self + .perpetual_tables + .owner_index + .insert(&(addr, object_ref.0), &ObjectInfo::new(&object_ref, object))?, + Owner::ObjectOwner(object_id) => { + if let Some(info) = + self.try_create_dynamic_field_info(&object_ref, object, Default::default()) + { + self.perpetual_tables + .dynamic_field_index + .insert(&(ObjectID::from(object_id), object_ref.0), &info)? + } + } + _ => {} + } // Only initialize lock for address owned objects. if !object.is_child_object() { self.lock_service @@ -567,7 +613,6 @@ impl AuthorityStore { .await?; } } - // Update the parent self.perpetual_tables .parent_sync @@ -576,6 +621,71 @@ impl AuthorityStore { Ok(()) } + fn try_create_dynamic_field_info( + &self, + oref: &ObjectRef, + o: &Object, + other_objects: BTreeMap, + ) -> Option { + let field = o.data.try_as_move()?; + if !is_dynamic_field(&field.type_) { + return None; + } + // Assuming dynamic field's bytearray structured as following [field object id][field name][1][object id] + // field object id and object id is the same for dynamic field, and different for dynamic object. + // index of the 1u8 byte + let (index, _) = field + .contents() + .iter() + .find_position(|byte| **byte == 1u8)?; + let name = &field.contents()[ObjectID::LENGTH..index]; + + let name = if name.len() == ObjectID::LENGTH { + ObjectID::try_from(name).ok()?.to_hex_literal() + } else if let Ok(name) = String::from_utf8(name.to_vec()) { + name + } else { + Base64::encode(name) + }; + + let object_id = + ObjectID::try_from(&field.contents()[index + 1..ObjectID::LENGTH + index + 1]).ok()?; + + Some(if is_dynamic_object_field(&field.type_.type_params[0]) { + let (object_type, version, digest) = if let Some(o) = other_objects.get(&object_id) { + o.data + .type_() + .cloned() + .map(|type_| (type_, o.version(), o.digest())) + } else if let Ok(Some(o)) = self.get_object(&object_id) { + o.data + .type_() + .cloned() + .map(|type_| (type_, o.version(), o.digest())) + } else { + warn!("Cannot get struct type for dynamic object {object_id}"); + None + }?; + DynamicFieldInfo { + name, + type_: DynamicFieldType::Object, + object_type: object_type.to_string(), + object_id, + version, + digest, + } + } else { + DynamicFieldInfo { + name, + type_: DynamicFieldType::Field(object_id), + object_type: field.type_.type_params[1].to_string(), + object_id: oref.0, + version: oref.1, + digest: oref.2, + } + }) + } + /// This function is used by the bench.rs script, and should not be used in other contexts /// In particular it does not check the old locks before inserting new ones, so the objects /// must be new. @@ -595,9 +705,31 @@ impl AuthorityStore { )? .insert_batch( &self.perpetual_tables.owner_index, - ref_and_objects - .iter() - .map(|(oref, o)| ((o.owner, oref.0), ObjectInfo::new(oref, o))), + ref_and_objects.iter().filter_map(|(oref, o)| { + if let Owner::AddressOwner(addr) = o.owner { + Some(((addr, oref.0), ObjectInfo::new(oref, o))) + } else { + None + } + }), + )? + .insert_batch( + &self.perpetual_tables.dynamic_field_index, + ref_and_objects.iter().filter_map(|(oref, o)| { + if let Owner::ObjectOwner(object_id) = o.owner { + self.try_create_dynamic_field_info( + oref, + o, + ref_and_objects + .iter() + .map(|((id, ..), o)| (*id, **o)) + .collect(), + ) + .map(|info| ((ObjectID::from(object_id), oref.0), info)) + } else { + None + } + }), )? .insert_batch( &self.perpetual_tables.parent_sync, @@ -813,7 +945,7 @@ impl AuthorityStore { // For wrapped objects, although their owners technically didn't change, we will lose track // of them and there is no guarantee on their owner in the future. Hence we treat them // the same as deleted. - let old_object_owners = + let (old_object_owners, old_dynamic_fields): (Vec<_>, Vec<_>) = deleted .iter() // We need to call get() on objects because some object that were just deleted may not @@ -827,11 +959,24 @@ impl AuthorityStore { } _ => None, }, - )); + )) + .map(|(owner, id)| match owner { + Owner::AddressOwner(address) => (Some((address, id)), None), + Owner::ObjectOwner(object_id) => (None, Some((ObjectID::from(object_id), id))), + _ => (None, None), + }) + .unzip(); + + let old_object_owners = old_object_owners.into_iter().flatten(); + let old_dynamic_fields = old_dynamic_fields.into_iter().flatten(); // Delete the old owner index entries write_batch = write_batch.delete_batch(&self.perpetual_tables.owner_index, old_object_owners)?; + write_batch = write_batch.delete_batch( + &self.perpetual_tables.dynamic_field_index, + old_dynamic_fields, + )?; // Index the certificate by the objects mutated write_batch = write_batch.insert_batch( @@ -861,18 +1006,35 @@ impl AuthorityStore { )?; // Update the indexes of the objects written + let (owner_written, dynamic_field_written): (Vec<_>, Vec<_>) = written + .iter() + .map(|(id, (object_ref, new_object, _))| match new_object.owner { + Owner::AddressOwner(address) => ( + Some(((address, *id), ObjectInfo::new(object_ref, new_object))), + None, + ), + Owner::ObjectOwner(object_id) => ( + None, + self.try_create_dynamic_field_info( + object_ref, + new_object, + written.iter().map(|(id, (_, o, _))| (*id, o)).collect(), + ) + .map(|info| ((ObjectID::from(object_id), *id), info)), + ), + _ => (None, None), + }) + .unzip(); + + let owner_written = owner_written.into_iter().flatten(); + let dynamic_field_written = dynamic_field_written.into_iter().flatten(); + + write_batch = + write_batch.insert_batch(&self.perpetual_tables.owner_index, owner_written)?; write_batch = write_batch.insert_batch( - &self.perpetual_tables.owner_index, - written - .iter() - .filter_map(|(_id, (object_ref, new_object, _kind))| { - trace!(?object_ref, owner =? new_object.owner, "Updating owner_index"); - new_object - .get_owner_and_id() - .map(|owner_id| (owner_id, ObjectInfo::new(object_ref, new_object))) - }), + &self.perpetual_tables.dynamic_field_index, + dynamic_field_written, )?; - // Insert each output object into the stores write_batch = write_batch.insert_batch( &self.perpetual_tables.objects, @@ -988,44 +1150,80 @@ impl AuthorityStore { // the older version, and then rewrite the entry with the old object info. // TODO: Validators should not need to maintain owner_index. // This is dependent on https://github.com/MystenLabs/sui/issues/2629. - let owners_to_delete = effects + let (owners_to_delete, dynamic_field_to_delete): (Vec<_>, Vec<_>) = effects .created .iter() .chain(effects.unwrapped.iter()) .chain(effects.mutated.iter()) - .map(|((id, _, _), owner)| (*owner, *id)); + .map(|((id, _, _), owner)| match owner { + Owner::AddressOwner(addr) => (Some((*addr, *id)), None), + Owner::ObjectOwner(object_id) => (None, Some((ObjectID::from(*object_id), *id))), + _ => (None, None), + }) + .unzip(); + + let owners_to_delete = owners_to_delete.into_iter().flatten(); + let dynamic_field_to_delete = dynamic_field_to_delete.into_iter().flatten(); + write_batch = write_batch.delete_batch(&self.perpetual_tables.owner_index, owners_to_delete)?; - + write_batch = write_batch.delete_batch( + &self.perpetual_tables.dynamic_field_index, + dynamic_field_to_delete, + )?; let modified_object_keys = effects .modified_at_versions .iter() - .map(|(id, version)| ObjectKey(*id, *version)); - - let (old_modified_objects, old_locks): (Vec<_>, Vec<_>) = self + .map(|(r, _)| r) + .chain(effects.deleted.iter()) + .chain(effects.wrapped.iter()) + .map(|(id, version, _)| { + ObjectKey( + *id, + version + .decrement() + .expect("version revert should never fail"), + ) + }); + let (old_objects_and_locks, old_dynamic_fields): (Vec<_>, Vec<_>) = self .perpetual_tables .objects .multi_get(modified_object_keys)? .into_iter() - .filter_map(|obj_opt| { + .map(|obj_opt| { let obj = obj_opt.expect("Older object version not found"); - - if obj.is_immutable() { - return None; + match obj.owner { + Owner::AddressOwner(addr) => { + let oref = obj.compute_object_reference(); + ( + Some((((addr, obj.id()), ObjectInfo::new(&oref, &obj)), oref)), + None, + ) + } + Owner::ObjectOwner(object_id) => ( + None, + self.try_create_dynamic_field_info( + &obj.compute_object_reference(), + &obj, + Default::default(), + ) + .map(|info| ((ObjectID::from(object_id), obj.id()), info)), + ), + _ => (None, None), } - - let obj_ref = obj.compute_object_reference(); - Some(( - ((obj.owner, obj.id()), ObjectInfo::new(&obj_ref, &obj)), - obj.is_address_owned().then_some(obj_ref), - )) }) .unzip(); - let old_locks: Vec<_> = old_locks.into_iter().flatten().collect(); + let (old_objects, old_locks): (Vec<_>, Vec<_>) = + old_objects_and_locks.into_iter().flatten().unzip(); + let old_dynamic_fields = old_dynamic_fields.into_iter().flatten(); write_batch = write_batch.insert_batch(&self.perpetual_tables.owner_index, old_modified_objects)?; + write_batch = write_batch.insert_batch( + &self.perpetual_tables.dynamic_field_index, + old_dynamic_fields, + )?; write_batch.write()?; @@ -1416,6 +1614,18 @@ impl ModuleResolver for AuthorityStore { } } +fn is_dynamic_field(tag: &StructTag) -> bool { + tag.address == SUI_FRAMEWORK_ADDRESS + && tag.module.as_str() == "dynamic_field" + && tag.name.as_str() == "Field" +} + +fn is_dynamic_object_field(tag: &TypeTag) -> bool { + matches!(tag, TypeTag::Struct(tag) if tag.address == SUI_FRAMEWORK_ADDRESS + && tag.module.as_str() == "dynamic_object_field" + && tag.name.as_str() == "Wrapper") +} + /// A wrapper to make Orphan Rule happy pub struct ResolverWrapper(pub Arc); diff --git a/crates/sui-core/src/authority/authority_store_tables.rs b/crates/sui-core/src/authority/authority_store_tables.rs index 63acc1cfe0d9e..d5d41fd3a751d 100644 --- a/crates/sui-core/src/authority/authority_store_tables.rs +++ b/crates/sui-core/src/authority/authority_store_tables.rs @@ -34,7 +34,9 @@ pub struct AuthorityPerpetualTables { /// composite key of the SuiAddress of their owner and the object ID of the object. /// This composite index allows an efficient iterator to list all objected currently owned /// by a specific user, and their object reference. - pub(crate) owner_index: DBMap<(Owner, ObjectID), ObjectInfo>, + pub(crate) owner_index: DBMap<(SuiAddress, ObjectID), ObjectInfo>, + + pub(crate) dynamic_field_index: DBMap<(ObjectID, ObjectID), DynamicFieldInfo>, /// This is a map between the transaction digest and the corresponding certificate for all /// certificates that have been successfully processed by this authority. This set of certificates diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index 8e82451fd6829..2adb87075ecf1 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -2455,16 +2455,8 @@ async fn test_store_revert_transfer_sui() { db.get_latest_parent_entry(gas_object_id).unwrap().unwrap(), (gas_object_ref, TransactionDigest::genesis()), ); - assert!(db - .get_owner_objects(Owner::AddressOwner(recipient)) - .unwrap() - .is_empty()); - assert_eq!( - db.get_owner_objects(Owner::AddressOwner(sender)) - .unwrap() - .len(), - 1 - ); + assert!(db.get_owner_objects(recipient).unwrap().is_empty()); + assert_eq!(db.get_owner_objects(sender).unwrap().len(), 1); assert!(db.get_certified_transaction(&tx_digest).unwrap().is_none()); assert!(db.as_ref().get_effects(&tx_digest).is_err()); } diff --git a/crates/sui-json-rpc-types/src/lib.rs b/crates/sui-json-rpc-types/src/lib.rs index ed19fcdd2163d..26b85647d0f77 100644 --- a/crates/sui-json-rpc-types/src/lib.rs +++ b/crates/sui-json-rpc-types/src/lib.rs @@ -28,6 +28,9 @@ use serde::Deserialize; use serde::Serialize; use serde_json::Value; use serde_with::serde_as; +use sui_types::coin::CoinMetadata; +use tracing::warn; + use sui_json::SuiJsonValue; use sui_types::base_types::{ AuthorityName, ObjectDigest, ObjectID, ObjectInfo, ObjectRef, SequenceNumber, SuiAddress, @@ -64,6 +67,7 @@ pub type SuiMoveTypeParameterIndex = u16; pub type TransactionsPage = Page; pub type EventPage = Page; pub type CoinPage = Page; +pub type DynamicFieldPage = Page; #[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(rename_all = "camelCase")] diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index 6be44e7b95b5b..ae346e1d7c6db 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -90,12 +90,12 @@ pub trait RpcReadApi { ) -> RpcResult>; /// Return the list of objects owned by an object. - #[method(name = "getObjectsOwnedByObject")] - async fn get_objects_owned_by_object( + #[method(name = "getDynamicFields")] + async fn get_dynamic_fields( &self, /// the ID of the owner object object_id: ObjectID, - ) -> RpcResult>; + ) -> RpcResult; /// Return the total number of transactions known to the server. #[method(name = "getTotalTransactionNumber")] diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index 09169dae98f03..fece0411bf927 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -6,20 +6,23 @@ use async_trait::async_trait; use jsonrpsee::core::RpcResult; use move_binary_format::normalized::{Module as NormalizedModule, Type}; use move_core_types::identifier::Identifier; +use move_core_types::language_storage::StructTag; +use signature::Signature; use std::collections::BTreeMap; use std::sync::Arc; use sui_types::intent::{Intent, IntentMessage}; use sui_types::sui_system_state::SuiSystemState; use tap::TapFallible; +use tracing::debug; use fastcrypto::encoding::Base64; use jsonrpsee::RpcModule; use sui_core::authority::AuthorityState; use sui_json_rpc_types::{ - GetObjectDataResponse, GetPastObjectDataResponse, MoveFunctionArgType, ObjectValueKind, Page, - SuiMoveNormalizedFunction, SuiMoveNormalizedModule, SuiMoveNormalizedStruct, SuiObjectInfo, - SuiTransactionAuthSignersResponse, SuiTransactionEffects, SuiTransactionResponse, - TransactionsPage, + DynamicFieldPage, GetObjectDataResponse, GetPastObjectDataResponse, MoveFunctionArgType, + ObjectValueKind, Page, SuiMoveNormalizedFunction, SuiMoveNormalizedModule, + SuiMoveNormalizedStruct, SuiObjectInfo, SuiTransactionAuthSignersResponse, + SuiTransactionEffects, SuiTransactionResponse, TransactionsPage, }; use sui_open_rpc::Module; use sui_types::base_types::SequenceNumber; @@ -29,11 +32,9 @@ use sui_types::committee::EpochId; use sui_types::crypto::sha3_hash; use sui_types::messages::{CommitteeInfoRequest, CommitteeInfoResponse}; use sui_types::move_package::normalize_modules; -use sui_types::object::{Data, ObjectRead, Owner}; +use sui_types::object::{Data, ObjectRead}; use sui_types::query::TransactionQuery; -use tracing::debug; - use crate::api::RpcFullNodeReadApiServer; use crate::api::{cap_page_limit, RpcReadApiServer}; use crate::SuiRpcModule; @@ -68,24 +69,23 @@ impl RpcReadApiServer for ReadApi { ) -> RpcResult> { Ok(self .state - .get_owner_objects(Owner::AddressOwner(address)) + .get_owner_objects(address) .map_err(|e| anyhow!("{e}"))? .into_iter() .map(SuiObjectInfo::from) .collect()) } - async fn get_objects_owned_by_object( - &self, - object_id: ObjectID, - ) -> RpcResult> { - Ok(self + async fn get_dynamic_fields(&self, object_id: ObjectID) -> RpcResult { + let data = self .state - .get_owner_objects(Owner::ObjectOwner(object_id.into())) - .map_err(|e| anyhow!("{e}"))? - .into_iter() - .map(SuiObjectInfo::from) - .collect()) + .get_dynamic_fields(object_id) + .map_err(|e| anyhow!("{e}"))?; + + Ok(DynamicFieldPage { + data, + next_cursor: None, + }) } async fn get_object(&self, object_id: ObjectID) -> RpcResult { @@ -380,3 +380,25 @@ pub async fn get_move_modules_by_package( _ => Err(anyhow!("Package object does not exist with ID {}", package)), }?) } + +/* +#[test] +fn test() { + let tag = "0x2::dynamic_field::Field<0x2::dynamic_object_field::Wrapper<0x2::typed_id::TypedID<0xdf496988c6d4571684d9f4edcaec5a1cf889fa2c::geniteam::Farm>>, 0x2::object::ID>"; + let tag2 = "0x2::dynamic_field::Field<0x2::object::ID, 0xdf496988c6d4571684d9f4edcaec5a1cf889fa2c::marketplace::Listing<0x2::devnet_nft::DevNetNFT, 0x2::sui::SUI>>"; + + let tag = parse_sui_struct_tag(tag).unwrap(); + let tag2 = parse_sui_struct_tag(tag2).unwrap(); + + for tag in [tag, tag2] { + if is_dynamic_field(&tag) { + if matches!(&tag.type_params[0], TypeTag::Struct(tag) if is_dynamic_field_wrapper(&tag)) + { + println!("1") + } else { + println!("2") + } + } + } +} +*/ diff --git a/crates/sui-json-rpc/src/transaction_builder_api.rs b/crates/sui-json-rpc/src/transaction_builder_api.rs index 3755a52911708..6e22dda58a9e4 100644 --- a/crates/sui-json-rpc/src/transaction_builder_api.rs +++ b/crates/sui-json-rpc/src/transaction_builder_api.rs @@ -11,7 +11,6 @@ use sui_json_rpc_types::{GetRawObjectDataResponse, SuiObjectInfo, SuiTypeTag, Tr use sui_open_rpc::Module; use sui_transaction_builder::{DataReader, TransactionBuilder}; use sui_types::base_types::{ObjectID, SuiAddress}; -use sui_types::object::Owner; use fastcrypto::encoding::Base64; use jsonrpsee::RpcModule; @@ -48,7 +47,7 @@ impl DataReader for AuthorityStateDataReader { ) -> Result, anyhow::Error> { let refs: Vec = self .0 - .get_owner_objects(Owner::AddressOwner(address))? + .get_owner_objects(address)? .into_iter() .map(SuiObjectInfo::from) .collect(); diff --git a/crates/sui-rosetta/src/account.rs b/crates/sui-rosetta/src/account.rs index 6f2c6477f8961..2c100724faabf 100644 --- a/crates/sui-rosetta/src/account.rs +++ b/crates/sui-rosetta/src/account.rs @@ -9,7 +9,6 @@ use axum::{Extension, Json}; use sui_core::authority::AuthorityState; use sui_types::base_types::SuiAddress; use sui_types::gas_coin::GasCoin; -use sui_types::object::Owner; use crate::errors::Error; use crate::types::{ @@ -72,7 +71,7 @@ pub async fn coins( } async fn get_coins(state: &AuthorityState, address: SuiAddress) -> Result, Error> { - let object_infos = state.get_owner_objects(Owner::AddressOwner(address))?; + let object_infos = state.get_owner_objects(address)?; let coin_infos = object_infos .iter() .filter(|o| o.type_.is_gas_coin()) diff --git a/crates/sui-rosetta/src/construction.rs b/crates/sui-rosetta/src/construction.rs index a07a51ebd7621..4777c8d72d1b8 100644 --- a/crates/sui-rosetta/src/construction.rs +++ b/crates/sui-rosetta/src/construction.rs @@ -13,7 +13,6 @@ use sui_types::messages::{ QuorumDriverRequest, QuorumDriverRequestType, QuorumDriverResponse, Transaction, TransactionData, }; -use sui_types::object::Owner; use crate::errors::Error; use crate::operations::Operation; @@ -219,7 +218,7 @@ pub async fn metadata( let sender_coins = if let Some(option) = request.options { context .state - .get_owner_objects(Owner::AddressOwner(option.sender))? + .get_owner_objects(option.sender)? .iter() .filter(|info| info.type_.is_gas_coin()) .map(|info| info.into()) diff --git a/crates/sui-sdk/src/lib.rs b/crates/sui-sdk/src/lib.rs index 65d73c938cc3d..91276e2bba34b 100644 --- a/crates/sui-sdk/src/lib.rs +++ b/crates/sui-sdk/src/lib.rs @@ -15,11 +15,17 @@ use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; use crate::error::{RpcError, SuiRpcResult}; use rpc_types::{SuiCertifiedTransaction, SuiParsedTransactionResponse, SuiTransactionEffects}; use serde_json::Value; +pub use sui_config::gateway; +use sui_core::gateway_state::TxSeqNumber; pub use sui_json as json; use crate::apis::{CoinReadApi, EventApi, QuorumDriver, ReadApi}; pub use sui_json_rpc_types as rpc_types; -use sui_json_rpc_types::{GetRawObjectDataResponse, SuiObjectInfo}; +use sui_json_rpc_types::{ + DynamicFieldPage, EventPage, GetObjectDataResponse, GetRawObjectDataResponse, SuiEventEnvelope, + SuiEventFilter, SuiMoveNormalizedModule, SuiObjectInfo, SuiTransactionResponse, + TransactionsPage, +}; use sui_transaction_builder::{DataReader, TransactionBuilder}; pub use sui_types as types; use sui_types::base_types::{ObjectID, SuiAddress, TransactionDigest}; diff --git a/crates/sui-types/src/base_types.rs b/crates/sui-types/src/base_types.rs index cc723412d15bd..cbac76a526a4c 100644 --- a/crates/sui-types/src/base_types.rs +++ b/crates/sui-types/src/base_types.rs @@ -5,10 +5,10 @@ use anyhow::anyhow; use curve25519_dalek::ristretto::RistrettoPoint; use fastcrypto::encoding::decode_bytes_hex; +use fastcrypto::encoding::{Base64, Encoding, Hex}; use move_core_types::account_address::AccountAddress; use move_core_types::ident_str; use move_core_types::identifier::IdentStr; -use move_core_types::language_storage::StructTag; use opentelemetry::{global, Context}; use rand::Rng; use schemars::JsonSchema; @@ -35,7 +35,9 @@ use crate::gas_coin::GasCoin; use crate::object::{Object, Owner}; use crate::sui_serde::Readable; use fastcrypto::encoding::{Base58, Base64, Encoding, Hex}; +use crate::SUI_FRAMEWORK_ADDRESS; use fastcrypto::hash::{HashFunction, Sha3_256}; +use move_core_types::language_storage::StructTag; #[cfg(test)] #[path = "unit_tests/base_types_tests.rs"] @@ -147,6 +149,23 @@ impl From<&ObjectInfo> for ObjectRef { } } +#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] +#[serde(rename_all = "camelCase")] +pub struct DynamicFieldInfo { + pub name: String, + pub type_: DynamicFieldType, + pub object_type: String, + pub object_id: ObjectID, + pub version: SequenceNumber, + pub digest: ObjectDigest, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] +pub enum DynamicFieldType { + Field(ObjectID), + Object, +} + pub const SUI_ADDRESS_LENGTH: usize = ObjectID::LENGTH; #[serde_as] diff --git a/crates/sui/src/client_commands.rs b/crates/sui/src/client_commands.rs index ea68fd167b2e6..c2ad326d56db4 100644 --- a/crates/sui/src/client_commands.rs +++ b/crates/sui/src/client_commands.rs @@ -686,12 +686,6 @@ impl SuiClientCommands { .read_api() .get_objects_owned_by_address(address) .await?; - let object_objects = client - .read_api() - .get_objects_owned_by_object(address.into()) - .await?; - address_object.extend(object_objects); - SuiClientCommandResult::Objects(address_object) } From 4b8fafa22284f93d4ce9ce3b28f922fbdb24c5e3 Mon Sep 17 00:00:00 2001 From: patrick Date: Tue, 22 Nov 2022 23:36:57 +0000 Subject: [PATCH 02/13] impl GetModule for SuiDataStore parse and process move object data for dynamic objects added test --- .../sui-core/src/authority/authority_store.rs | 188 ++++++++++++------ crates/sui-core/src/lib.rs | 2 + .../src/unit_tests/authority_tests.rs | 166 ++++++++++++++-- .../object_basics/sources/object_basics.move | 11 + crates/sui-types/src/base_types.rs | 7 +- crates/sui-types/src/object.rs | 2 +- 6 files changed, 305 insertions(+), 71 deletions(-) diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index 84fe1cf2eb7e7..f1533b6238962 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -12,9 +12,10 @@ use crate::authority::authority_per_epoch_store::{ AuthorityPerEpochStore, ExecutionIndicesWithHash, }; use arc_swap::ArcSwap; -use fastcrypto::encoding::{Base64, Encoding}; -use itertools::Itertools; +use move_binary_format::CompiledModule; +use move_bytecode_utils::module_cache::GetModule; use move_core_types::language_storage::{StructTag, TypeTag}; +use move_core_types::value::{MoveStruct, MoveValue}; use once_cell::sync::OnceCell; use rocksdb::Options; use serde::{Deserialize, Serialize}; @@ -74,6 +75,8 @@ pub struct AuthorityStore { // Implementation detail to support notify_read_effects(). pub(crate) effects_notify_read: NotifyRead, + + module_cache: RwLock>>, } impl AuthorityStore { @@ -137,6 +140,7 @@ impl AuthorityStore { path: path.into(), db_options, effects_notify_read: NotifyRead::new(), + module_cache: Default::default(), }; // Only initialize an empty database. if store @@ -627,63 +631,88 @@ impl AuthorityStore { o: &Object, other_objects: BTreeMap, ) -> Option { - let field = o.data.try_as_move()?; - if !is_dynamic_field(&field.type_) { + let move_object = o.data.try_as_move()?; + if !is_dynamic_field(&move_object.type_) { return None; } - // Assuming dynamic field's bytearray structured as following [field object id][field name][1][object id] - // field object id and object id is the same for dynamic field, and different for dynamic object. - // index of the 1u8 byte - let (index, _) = field - .contents() - .iter() - .find_position(|byte| **byte == 1u8)?; - let name = &field.contents()[ObjectID::LENGTH..index]; - - let name = if name.len() == ObjectID::LENGTH { - ObjectID::try_from(name).ok()?.to_hex_literal() - } else if let Ok(name) = String::from_utf8(name.to_vec()) { - name - } else { - Base64::encode(name) - }; - - let object_id = - ObjectID::try_from(&field.contents()[index + 1..ObjectID::LENGTH + index + 1]).ok()?; - - Some(if is_dynamic_object_field(&field.type_.type_params[0]) { - let (object_type, version, digest) = if let Some(o) = other_objects.get(&object_id) { - o.data - .type_() - .cloned() - .map(|type_| (type_, o.version(), o.digest())) - } else if let Ok(Some(o)) = self.get_object(&object_id) { - o.data - .type_() - .cloned() - .map(|type_| (type_, o.version(), o.digest())) + let move_struct = move_object + .to_move_struct_with_resolver( + ObjectFormatOptions { + include_types: false, + }, + &self, + ) + .ok()?; + + let name = extract_field_from_move_struct(&move_struct, "name") + .expect("Expecting [name] field in sui::dynamic_field::Field"); + let value = extract_field_from_move_struct(&move_struct, "value") + .expect("Expecting [value] field in sui::dynamic_field::Field"); + // Extract value from value option + let value = match value { + MoveValue::Struct(MoveStruct::WithFields(fields)) => match fields.first() { + Some((_, MoveValue::Vector(v))) => v.first().cloned(), + _ => None, + }, + _ => None, + }?; + + Some( + if is_dynamic_object_field(&move_object.type_.type_params[0]) { + let name = match name { + MoveValue::Struct(s) => extract_field_from_move_struct(&s, "name"), + _ => None, + } + .expect("Expecting [name] field in sui::dynamic_object_field::Wrapper."); + + let object_id = extract_object_id(&value) + .expect("Cannot extract dynamic object's object id from Field::value"); + + let (object_type, version, digest) = if let Some(o) = other_objects.get(&object_id) + { + o.data + .type_() + .cloned() + .map(|type_| (type_, o.version(), o.digest())) + } else if let Ok(Some(o)) = self.get_object(&object_id) { + o.data + .type_() + .cloned() + .map(|type_| (type_, o.version(), o.digest())) + } else { + warn!("Cannot get struct type for dynamic object {object_id}"); + None + }?; + DynamicFieldInfo { + name: name.to_string(), + type_: DynamicFieldType::DynamicObject, + object_type: object_type.to_string(), + object_id, + version, + digest, + } } else { - warn!("Cannot get struct type for dynamic object {object_id}"); - None - }?; - DynamicFieldInfo { - name, - type_: DynamicFieldType::Object, - object_type: object_type.to_string(), - object_id, - version, - digest, - } - } else { - DynamicFieldInfo { - name, - type_: DynamicFieldType::Field(object_id), - object_type: field.type_.type_params[1].to_string(), - object_id: oref.0, - version: oref.1, - digest: oref.2, - } - }) + let wrapped_object_id = match value { + // UID + MoveValue::Struct(s) => match extract_field_from_move_struct(&s, "id") { + // ID + Some(MoveValue::Struct(s)) => extract_field_from_move_struct(&s, "id") + .and_then(|v| extract_object_id(&v)), + _ => None, + }, + _ => None, + } + .expect("Cannot extract dynamic object's object id from Field::value"); + DynamicFieldInfo { + name: name.to_string(), + type_: DynamicFieldType::DynamicField { wrapped_object_id }, + object_type: move_object.type_.type_params[1].to_string(), + object_id: oref.0, + version: oref.1, + digest: oref.2, + } + }, + ) } /// This function is used by the bench.rs script, and should not be used in other contexts @@ -1614,6 +1643,53 @@ impl ModuleResolver for AuthorityStore { } } +impl Deserialize<'de>> GetModule for &SuiDataStore { + type Error = SuiError; + type Item = Arc; + + fn get_module_by_id(&self, id: &ModuleId) -> Result>, Self::Error> { + if let Some(compiled_module) = self.module_cache.read().get(id) { + return Ok(Some(compiled_module.clone())); + } + + if let Some(module_bytes) = self.get_module(id)? { + let module = Arc::new(CompiledModule::deserialize(&module_bytes).map_err(|e| { + SuiError::ModuleDeserializationFailure { + error: e.to_string(), + } + })?); + + self.module_cache.write().insert(id.clone(), module.clone()); + Ok(Some(module)) + } else { + Ok(None) + } + } +} + +fn extract_field_from_move_struct(move_struct: &MoveStruct, field_name: &str) -> Option { + match move_struct { + MoveStruct::WithFields(fields) => fields.iter().find_map(|(id, value)| { + if id.to_string() == field_name { + Some(value.clone()) + } else { + None + } + }), + _ => None, + } +} + +fn extract_object_id(value: &MoveValue) -> Option { + match value { + MoveValue::Struct(MoveStruct::WithFields(fields)) => match fields.first() { + Some((_, MoveValue::Address(addr))) => Some(ObjectID::from(*addr)), + _ => None, + }, + _ => None, + } +} + fn is_dynamic_field(tag: &StructTag) -> bool { tag.address == SUI_FRAMEWORK_ADDRESS && tag.module.as_str() == "dynamic_field" diff --git a/crates/sui-core/src/lib.rs b/crates/sui-core/src/lib.rs index 4db074ad82fdd..0354f34855530 100644 --- a/crates/sui-core/src/lib.rs +++ b/crates/sui-core/src/lib.rs @@ -1,3 +1,5 @@ +extern crate core; + // Copyright (c) 2021, Facebook, Inc. and its affiliates // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index 2adb87075ecf1..72c746edef214 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -2616,6 +2616,149 @@ async fn test_store_revert_unwrap_move_call() { let gas = db.get_object(&gas_object_id).unwrap().unwrap(); assert_eq!(gas.version(), wrap_effects.gas_object.0 .1); } +#[tokio::test] +async fn test_store_get_dynamic_object() { + let (sender, sender_key): (_, AccountKeyPair) = get_key_pair(); + let gas_object_id = ObjectID::random(); + let (authority_state, object_basics) = + init_state_with_ids_and_object_basics(vec![(sender, gas_object_id)]).await; + + let create_outer_effects = create_move_object( + &object_basics, + &authority_state, + &gas_object_id, + &sender, + &sender_key, + ) + .await + .unwrap(); + + assert!(create_outer_effects.status.is_ok()); + assert_eq!(create_outer_effects.created.len(), 1); + + let create_inner_effects = create_move_object( + &object_basics, + &authority_state, + &gas_object_id, + &sender, + &sender_key, + ) + .await + .unwrap(); + + assert!(create_inner_effects.status.is_ok()); + assert_eq!(create_inner_effects.created.len(), 1); + + let outer_v0 = create_outer_effects.created[0].0; + let inner_v0 = create_inner_effects.created[0].0; + + let add_txn = to_sender_signed_transaction( + TransactionData::new_move_call( + sender, + object_basics, + ident_str!("object_basics").to_owned(), + ident_str!("add_ofield").to_owned(), + vec![], + create_inner_effects.gas_object.0, + vec![ + CallArg::Object(ObjectArg::ImmOrOwnedObject(outer_v0)), + CallArg::Object(ObjectArg::ImmOrOwnedObject(inner_v0)), + ], + MAX_GAS, + ), + &sender_key, + ); + + let add_cert = init_certified_transaction(add_txn, &authority_state); + + let add_effects = authority_state + .handle_certificate(&add_cert) + .await + .unwrap() + .signed_effects + .unwrap() + .into_data(); + + assert!(add_effects.status.is_ok()); + assert_eq!(add_effects.created.len(), 1); + + let fields = authority_state.get_dynamic_fields(outer_v0.0).unwrap(); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].type_, DynamicFieldType::DynamicObject); +} + +#[tokio::test] +async fn test_store_get_dynamic_field() { + let (sender, sender_key): (_, AccountKeyPair) = get_key_pair(); + let gas_object_id = ObjectID::random(); + let (authority_state, object_basics) = + init_state_with_ids_and_object_basics(vec![(sender, gas_object_id)]).await; + + let create_outer_effects = create_move_object( + &object_basics, + &authority_state, + &gas_object_id, + &sender, + &sender_key, + ) + .await + .unwrap(); + + assert!(create_outer_effects.status.is_ok()); + assert_eq!(create_outer_effects.created.len(), 1); + + let create_inner_effects = create_move_object( + &object_basics, + &authority_state, + &gas_object_id, + &sender, + &sender_key, + ) + .await + .unwrap(); + + assert!(create_inner_effects.status.is_ok()); + assert_eq!(create_inner_effects.created.len(), 1); + + let outer_v0 = create_outer_effects.created[0].0; + let inner_v0 = create_inner_effects.created[0].0; + + let add_txn = to_sender_signed_transaction( + TransactionData::new_move_call( + sender, + object_basics, + ident_str!("object_basics").to_owned(), + ident_str!("add_field").to_owned(), + vec![], + create_inner_effects.gas_object.0, + vec![ + CallArg::Object(ObjectArg::ImmOrOwnedObject(outer_v0)), + CallArg::Object(ObjectArg::ImmOrOwnedObject(inner_v0)), + ], + MAX_GAS, + ), + &sender_key, + ); + + let add_cert = init_certified_transaction(add_txn, &authority_state); + + let add_effects = authority_state + .handle_certificate(&add_cert) + .await + .unwrap() + .signed_effects + .unwrap() + .into_data(); + + assert!(add_effects.status.is_ok()); + assert_eq!(add_effects.created.len(), 1); + + let fields = authority_state.get_dynamic_fields(outer_v0.0).unwrap(); + assert_eq!(fields.len(), 1); + assert!( + matches!(fields[0].type_, DynamicFieldType::DynamicField {wrapped_object_id} if wrapped_object_id == inner_v0.0) + ); +} #[tokio::test] async fn test_store_revert_add_ofield() { @@ -2869,17 +3012,19 @@ pub async fn init_state_with_ids_and_object_basics< >( objects: I, ) -> (Arc, ObjectRef) { - use sui_framework_build::compiled_package::BuildConfig; + let state = init_state_with_ids(objects).await; + let pkg = build_package("src/unit_tests/data/object_basics"); + let pkg_ref = pkg.compute_object_reference(); + state.insert_genesis_object(pkg).await; + (state, pkg_ref) +} - let state = init_state().await; - for (address, object_id) in objects { - let obj = Object::with_id_owner_for_testing(object_id, address); - state.insert_genesis_object(obj).await; - } +#[cfg(test)] +pub fn build_package(package_path: &str) -> Object { + use sui_framework_build::compiled_package::BuildConfig; - // add object_basics package object to genesis, since lots of test use it let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push("src/unit_tests/data/object_basics"); + path.push(package_path); let modules = BuildConfig::default() .build(path) .unwrap() @@ -2887,10 +3032,7 @@ pub async fn init_state_with_ids_and_object_basics< .into_iter() .cloned() .collect(); - let pkg = Object::new_package(modules, TransactionDigest::genesis()).unwrap(); - let pkg_ref = pkg.compute_object_reference(); - state.insert_genesis_object(pkg).await; - (state, pkg_ref) + Object::new_package(modules, TransactionDigest::genesis().unwrap()) } #[cfg(test)] diff --git a/crates/sui-core/src/unit_tests/data/object_basics/sources/object_basics.move b/crates/sui-core/src/unit_tests/data/object_basics/sources/object_basics.move index 283d256603582..70c7169755cfc 100644 --- a/crates/sui-core/src/unit_tests/data/object_basics/sources/object_basics.move +++ b/crates/sui-core/src/unit_tests/data/object_basics/sources/object_basics.move @@ -90,4 +90,15 @@ module examples::object_basics { fun get_contents(o: &Object): (ID, u64) { (object::id(o), o.value) } + + public entry fun add_field(o: &mut Object, v: Object) { + sui::dynamic_field::add(&mut o.id, true, v); + } + + public entry fun remove_field(o: &mut Object, ctx: &mut TxContext) { + transfer::transfer( + sui::dynamic_field::remove(&mut o.id, true), + tx_context::sender(ctx), + ); + } } diff --git a/crates/sui-types/src/base_types.rs b/crates/sui-types/src/base_types.rs index cbac76a526a4c..2613fe7a7cdc9 100644 --- a/crates/sui-types/src/base_types.rs +++ b/crates/sui-types/src/base_types.rs @@ -162,8 +162,11 @@ pub struct DynamicFieldInfo { #[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] pub enum DynamicFieldType { - Field(ObjectID), - Object, + #[serde(rename_all = "camelCase")] + DynamicField { + wrapped_object_id: ObjectID, + }, + DynamicObject, } pub const SUI_ADDRESS_LENGTH: usize = ObjectID::LENGTH; diff --git a/crates/sui-types/src/object.rs b/crates/sui-types/src/object.rs index 05159ea772a91..a8cf4d50baa5f 100644 --- a/crates/sui-types/src/object.rs +++ b/crates/sui-types/src/object.rs @@ -57,7 +57,7 @@ pub struct ObjectFormatOptions { /// `{ "fields": { "f": 20, "g": { "fields" { "h": true }, "type": "0x0::MyModule::MyNestedType" }, "type": "0x0::MyModule::MyType" }` /// If false, include field names only; e.g.: /// `{ "f": 20, "g": { "h": true } }` - include_types: bool, + pub include_types: bool, } impl MoveObject { From 3ee9c41af80d4982ea6b7b9de33c05b7fcfe6c87 Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Nov 2022 17:41:13 +0000 Subject: [PATCH 03/13] improve error handling --- crates/sui-core/src/authority.rs | 2 + .../sui-core/src/authority/authority_store.rs | 273 ++++++++---------- .../src/authority/authority_store_tables.rs | 1 + .../src/unit_tests/authority_tests.rs | 2 + crates/sui-json-rpc-types/src/lib.rs | 3 +- crates/sui-types/src/base_types.rs | 20 -- crates/sui-types/src/dynamic_field.rs | 144 +++++++++ crates/sui-types/src/error.rs | 2 + crates/sui-types/src/lib.rs | 1 + 9 files changed, 267 insertions(+), 181 deletions(-) create mode 100644 crates/sui-types/src/dynamic_field.rs diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index 71169ab863e7d..5fea7030cdc92 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -98,6 +98,8 @@ use crate::{ transaction_manager::TransactionManager, transaction_streamer::TransactionStreamer, }; +use narwhal_types::ConsensusOutput; +use sui_types::dynamic_field::DynamicFieldInfo; use sui_types::gas::GasCostSummary; diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index f1533b6238962..eecae0fa81067 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -14,8 +14,6 @@ use crate::authority::authority_per_epoch_store::{ use arc_swap::ArcSwap; use move_binary_format::CompiledModule; use move_bytecode_utils::module_cache::GetModule; -use move_core_types::language_storage::{StructTag, TypeTag}; -use move_core_types::value::{MoveStruct, MoveValue}; use once_cell::sync::OnceCell; use rocksdb::Options; use serde::{Deserialize, Serialize}; @@ -34,6 +32,9 @@ use sui_storage::{ mutex_table::{LockGuard, MutexTable}, LockService, }; +use sui_types::crypto::AuthoritySignInfo; +use sui_types::dynamic_field::{DynamicFieldInfo, DynamicFieldType}; +use sui_types::message_envelope::VerifiedEnvelope; use sui_types::object::Owner; use sui_types::storage::ChildObjectResolver; use sui_types::{base_types::SequenceNumber, storage::ParentSync}; @@ -600,9 +601,11 @@ impl AuthorityStore { .owner_index .insert(&(addr, object_ref.0), &ObjectInfo::new(&object_ref, object))?, Owner::ObjectOwner(object_id) => { - if let Some(info) = - self.try_create_dynamic_field_info(&object_ref, object, Default::default()) - { + if let Some(info) = self.try_create_dynamic_field_info( + &object_ref, + object, + &Default::default(), + )? { self.perpetual_tables .dynamic_field_index .insert(&(ObjectID::from(object_id), object_ref.0), &info)? @@ -629,90 +632,59 @@ impl AuthorityStore { &self, oref: &ObjectRef, o: &Object, - other_objects: BTreeMap, - ) -> Option { - let move_object = o.data.try_as_move()?; - if !is_dynamic_field(&move_object.type_) { - return None; + uncommitted_objects: &BTreeMap, + ) -> SuiResult> { + // Skip if not a move object + let move_object = if let Some(move_object) = o.data.try_as_move() { + move_object + } else { + return Ok(None); + }; + // We only index dynamic field objects + if !DynamicFieldInfo::is_dynamic_field(&move_object.type_) { + return Ok(None); } - let move_struct = move_object - .to_move_struct_with_resolver( - ObjectFormatOptions { - include_types: false, - }, - &self, - ) - .ok()?; - - let name = extract_field_from_move_struct(&move_struct, "name") - .expect("Expecting [name] field in sui::dynamic_field::Field"); - let value = extract_field_from_move_struct(&move_struct, "value") - .expect("Expecting [value] field in sui::dynamic_field::Field"); - // Extract value from value option - let value = match value { - MoveValue::Struct(MoveStruct::WithFields(fields)) => match fields.first() { - Some((_, MoveValue::Vector(v))) => v.first().cloned(), - _ => None, - }, - _ => None, - }?; - - Some( - if is_dynamic_object_field(&move_object.type_.type_params[0]) { - let name = match name { - MoveValue::Struct(s) => extract_field_from_move_struct(&s, "name"), - _ => None, - } - .expect("Expecting [name] field in sui::dynamic_object_field::Wrapper."); - - let object_id = extract_object_id(&value) - .expect("Cannot extract dynamic object's object id from Field::value"); - - let (object_type, version, digest) = if let Some(o) = other_objects.get(&object_id) - { - o.data - .type_() - .cloned() - .map(|type_| (type_, o.version(), o.digest())) - } else if let Ok(Some(o)) = self.get_object(&object_id) { - o.data - .type_() - .cloned() - .map(|type_| (type_, o.version(), o.digest())) - } else { - warn!("Cannot get struct type for dynamic object {object_id}"); - None - }?; + let move_struct = + move_object.to_move_struct_with_resolver(ObjectFormatOptions::default(), &self)?; + + let (name, type_, object_id) = DynamicFieldInfo::parse_move_object(&move_struct)?; + Ok(Some(match type_ { + DynamicFieldType::DynamicObject => { + // Find the actual object from storage using the object id obtained from the wrapper. + let (object_type, version, digest) = + if let Some(o) = uncommitted_objects.get(&object_id) { + o.data + .type_() + .map(|type_| (type_.clone(), o.version(), o.digest())) + } else if let Ok(Some(o)) = self.get_object(&object_id) { + o.data + .type_() + .map(|type_| (type_.clone(), o.version(), o.digest())) + } else { + None + } + .ok_or_else(|| SuiError::ObjectDeserializationError { + error: format!("Cannot found data for dynamic object {object_id}"), + })?; + DynamicFieldInfo { - name: name.to_string(), - type_: DynamicFieldType::DynamicObject, + name, + type_, object_type: object_type.to_string(), object_id, version, digest, } - } else { - let wrapped_object_id = match value { - // UID - MoveValue::Struct(s) => match extract_field_from_move_struct(&s, "id") { - // ID - Some(MoveValue::Struct(s)) => extract_field_from_move_struct(&s, "id") - .and_then(|v| extract_object_id(&v)), - _ => None, - }, - _ => None, - } - .expect("Cannot extract dynamic object's object id from Field::value"); - DynamicFieldInfo { - name: name.to_string(), - type_: DynamicFieldType::DynamicField { wrapped_object_id }, - object_type: move_object.type_.type_params[1].to_string(), - object_id: oref.0, - version: oref.1, - digest: oref.2, - } + } + DynamicFieldType::DynamicField { .. } => DynamicFieldInfo { + name, + type_, + object_type: move_object.type_.type_params[1].to_string(), + object_id: oref.0, + version: oref.1, + digest: oref.2, }, - ) + })) } /// This function is used by the bench.rs script, and should not be used in other contexts @@ -725,6 +697,38 @@ impl AuthorityStore { .map(|o| (o.compute_object_reference(), o)) .collect(); + let (owner_insert, dynamic_field_insert): (Vec<_>, Vec<_>) = ref_and_objects + .iter() + .map(|(object_ref, o)| match o.owner { + Owner::AddressOwner(address) => ( + Some(((address, o.id()), ObjectInfo::new(object_ref, o))), + None, + ), + Owner::ObjectOwner(object_id) => ( + None, + Some(((ObjectID::from(object_id), o.id()), (object_ref, o))), + ), + _ => (None, None), + }) + .unzip(); + + let owner_insert = owner_insert.into_iter().flatten(); + + let uncommitted_objects = ref_and_objects + .iter() + .map(|((id, ..), o)| (*id, **o)) + .collect(); + + let dynamic_field_insert = dynamic_field_insert + .into_iter() + .flatten() + .flat_map(|(key, (oref, o))| { + self.try_create_dynamic_field_info(oref, o, &uncommitted_objects) + .transpose() + .map(|info| info.map(|info| (key, info))) + }) + .collect::>>()?; + batch .insert_batch( &self.perpetual_tables.objects, @@ -732,33 +736,10 @@ impl AuthorityStore { .iter() .map(|(oref, o)| (ObjectKey::from(oref), **o)), )? - .insert_batch( - &self.perpetual_tables.owner_index, - ref_and_objects.iter().filter_map(|(oref, o)| { - if let Owner::AddressOwner(addr) = o.owner { - Some(((addr, oref.0), ObjectInfo::new(oref, o))) - } else { - None - } - }), - )? + .insert_batch(&self.perpetual_tables.owner_index, owner_insert)? .insert_batch( &self.perpetual_tables.dynamic_field_index, - ref_and_objects.iter().filter_map(|(oref, o)| { - if let Owner::ObjectOwner(object_id) = o.owner { - self.try_create_dynamic_field_info( - oref, - o, - ref_and_objects - .iter() - .map(|((id, ..), o)| (*id, **o)) - .collect(), - ) - .map(|info| ((ObjectID::from(object_id), oref.0), info)) - } else { - None - } - }), + dynamic_field_insert, )? .insert_batch( &self.perpetual_tables.parent_sync, @@ -1033,7 +1014,7 @@ impl AuthorityStore { ) }), )?; - + let uncommitted_objects = written.iter().map(|(id, (_, o, _))| (*id, o)).collect(); // Update the indexes of the objects written let (owner_written, dynamic_field_written): (Vec<_>, Vec<_>) = written .iter() @@ -1044,19 +1025,22 @@ impl AuthorityStore { ), Owner::ObjectOwner(object_id) => ( None, - self.try_create_dynamic_field_info( - object_ref, - new_object, - written.iter().map(|(id, (_, o, _))| (*id, o)).collect(), - ) - .map(|info| ((ObjectID::from(object_id), *id), info)), + Some(((ObjectID::from(object_id), *id), (object_ref, new_object))), ), _ => (None, None), }) .unzip(); let owner_written = owner_written.into_iter().flatten(); - let dynamic_field_written = dynamic_field_written.into_iter().flatten(); + let dynamic_field_written = dynamic_field_written + .into_iter() + .flatten() + .flat_map(|(key, (oref, o))| { + self.try_create_dynamic_field_info(oref, o, &uncommitted_objects) + .transpose() + .map(|info| info.map(|info| (key, info))) + }) + .collect::>>()?; write_batch = write_batch.insert_batch(&self.perpetual_tables.owner_index, owner_written)?; @@ -1229,15 +1213,9 @@ impl AuthorityStore { None, ) } - Owner::ObjectOwner(object_id) => ( - None, - self.try_create_dynamic_field_info( - &obj.compute_object_reference(), - &obj, - Default::default(), - ) - .map(|info| ((ObjectID::from(object_id), obj.id()), info)), - ), + Owner::ObjectOwner(object_id) => { + (None, Some(((ObjectID::from(object_id), obj.id()), obj))) + } _ => (None, None), } }) @@ -1245,7 +1223,19 @@ impl AuthorityStore { let (old_objects, old_locks): (Vec<_>, Vec<_>) = old_objects_and_locks.into_iter().flatten().unzip(); - let old_dynamic_fields = old_dynamic_fields.into_iter().flatten(); + let old_dynamic_fields = old_dynamic_fields + .into_iter() + .flatten() + .flat_map(|(key, o)| { + self.try_create_dynamic_field_info( + &o.compute_object_reference(), + &o, + &Default::default(), + ) + .transpose() + .map(|info| info.map(|info| (key, info))) + }) + .collect::>>()?; write_batch = write_batch.insert_batch(&self.perpetual_tables.owner_index, old_modified_objects)?; @@ -1667,41 +1657,6 @@ impl Deserialize<'de>> GetModule for &SuiDa } } -fn extract_field_from_move_struct(move_struct: &MoveStruct, field_name: &str) -> Option { - match move_struct { - MoveStruct::WithFields(fields) => fields.iter().find_map(|(id, value)| { - if id.to_string() == field_name { - Some(value.clone()) - } else { - None - } - }), - _ => None, - } -} - -fn extract_object_id(value: &MoveValue) -> Option { - match value { - MoveValue::Struct(MoveStruct::WithFields(fields)) => match fields.first() { - Some((_, MoveValue::Address(addr))) => Some(ObjectID::from(*addr)), - _ => None, - }, - _ => None, - } -} - -fn is_dynamic_field(tag: &StructTag) -> bool { - tag.address == SUI_FRAMEWORK_ADDRESS - && tag.module.as_str() == "dynamic_field" - && tag.name.as_str() == "Field" -} - -fn is_dynamic_object_field(tag: &TypeTag) -> bool { - matches!(tag, TypeTag::Struct(tag) if tag.address == SUI_FRAMEWORK_ADDRESS - && tag.module.as_str() == "dynamic_object_field" - && tag.name.as_str() == "Wrapper") -} - /// A wrapper to make Orphan Rule happy pub struct ResolverWrapper(pub Arc); diff --git a/crates/sui-core/src/authority/authority_store_tables.rs b/crates/sui-core/src/authority/authority_store_tables.rs index d5d41fd3a751d..bce5f0a21faea 100644 --- a/crates/sui-core/src/authority/authority_store_tables.rs +++ b/crates/sui-core/src/authority/authority_store_tables.rs @@ -7,6 +7,7 @@ use std::path::Path; use sui_storage::default_db_options; use sui_types::base_types::{ExecutionDigests, SequenceNumber}; use sui_types::batch::{SignedBatch, TxSequenceNumber}; +use sui_types::dynamic_field::DynamicFieldInfo; use sui_types::messages::TrustedCertificate; use typed_store::rocks::{DBMap, DBOptions}; use typed_store::traits::TypedStoreDebug; diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index 72c746edef214..5cfa8c1a7c45c 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -43,6 +43,8 @@ use sui_types::{ }; use sui_types::{crypto::AuthorityPublicKeyBytes, object::Data}; +use narwhal_types::Certificate; +use sui_types::dynamic_field::DynamicFieldType; use tracing::info; pub enum TestCallArg { diff --git a/crates/sui-json-rpc-types/src/lib.rs b/crates/sui-json-rpc-types/src/lib.rs index 26b85647d0f77..855e8e97e324c 100644 --- a/crates/sui-json-rpc-types/src/lib.rs +++ b/crates/sui-json-rpc-types/src/lib.rs @@ -28,8 +28,6 @@ use serde::Deserialize; use serde::Serialize; use serde_json::Value; use serde_with::serde_as; -use sui_types::coin::CoinMetadata; -use tracing::warn; use sui_json::SuiJsonValue; use sui_types::base_types::{ @@ -39,6 +37,7 @@ use sui_types::base_types::{ use sui_types::coin::CoinMetadata; use sui_types::committee::EpochId; use sui_types::crypto::{AuthorityStrongQuorumSignInfo, Signature}; +use sui_types::dynamic_field::DynamicFieldInfo; use sui_types::error::{ExecutionError, SuiError}; use sui_types::event::{BalanceChangeType, Event, EventID}; use sui_types::event::{EventEnvelope, EventType}; diff --git a/crates/sui-types/src/base_types.rs b/crates/sui-types/src/base_types.rs index 2613fe7a7cdc9..5d0a59d017b24 100644 --- a/crates/sui-types/src/base_types.rs +++ b/crates/sui-types/src/base_types.rs @@ -149,26 +149,6 @@ impl From<&ObjectInfo> for ObjectRef { } } -#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] -#[serde(rename_all = "camelCase")] -pub struct DynamicFieldInfo { - pub name: String, - pub type_: DynamicFieldType, - pub object_type: String, - pub object_id: ObjectID, - pub version: SequenceNumber, - pub digest: ObjectDigest, -} - -#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] -pub enum DynamicFieldType { - #[serde(rename_all = "camelCase")] - DynamicField { - wrapped_object_id: ObjectID, - }, - DynamicObject, -} - pub const SUI_ADDRESS_LENGTH: usize = ObjectID::LENGTH; #[serde_as] diff --git a/crates/sui-types/src/dynamic_field.rs b/crates/sui-types/src/dynamic_field.rs new file mode 100644 index 0000000000000..b20cc61bf5381 --- /dev/null +++ b/crates/sui-types/src/dynamic_field.rs @@ -0,0 +1,144 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use move_core_types::language_storage::{StructTag, TypeTag}; +use move_core_types::value::{MoveStruct, MoveValue}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; + +use crate::base_types::ObjectDigest; +use crate::error::{SuiError, SuiResult}; +use crate::id::ID; +use crate::{ObjectID, SequenceNumber, SUI_FRAMEWORK_ADDRESS}; + +#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] +#[serde(rename_all = "camelCase")] +pub struct DynamicFieldInfo { + pub name: String, + pub type_: DynamicFieldType, + pub object_type: String, + pub object_id: ObjectID, + pub version: SequenceNumber, + pub digest: ObjectDigest, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema, Ord, PartialOrd, Eq, PartialEq, Debug)] +pub enum DynamicFieldType { + #[serde(rename_all = "camelCase")] + DynamicField { + wrapped_object_id: ObjectID, + }, + DynamicObject, +} + +impl DynamicFieldInfo { + pub fn is_dynamic_field(tag: &StructTag) -> bool { + tag.address == SUI_FRAMEWORK_ADDRESS + && tag.module.as_str() == "dynamic_field" + && tag.name.as_str() == "Field" + } + + pub fn parse_move_object( + move_struct: &MoveStruct, + ) -> SuiResult<(String, DynamicFieldType, ObjectID)> { + let name = extract_field_from_move_struct(&move_struct, "name").ok_or_else(|| { + SuiError::ObjectDeserializationError { + error: "Cannot extract [name] field from sui::dynamic_field::Field".to_string(), + } + })?; + + let value = extract_field_from_move_struct(&move_struct, "value").ok_or_else(|| { + SuiError::ObjectDeserializationError { + error: "Cannot extract [value] field from sui::dynamic_field::Field".to_string(), + } + })?; + + // Extract value from value option + let value = match &value { + MoveValue::Struct(MoveStruct::WithTypes { type_: _, fields }) => match fields.first() { + Some((_, MoveValue::Vector(v))) => v.first().cloned(), + _ => None, + }, + _ => None, + } + .ok_or_else(|| SuiError::ObjectDeserializationError { + error: "Cannot extract optional value".to_string(), + })?; + + let object_id = + extract_object_id(&value).ok_or_else(|| SuiError::ObjectDeserializationError { + error: format!( + "Cannot extract dynamic object's object id from Field::value, {:?}", + value + ), + })?; + + Ok(if is_dynamic_object(move_struct) { + let name = match name { + MoveValue::Struct(s) => extract_field_from_move_struct(&s, "name"), + _ => None, + } + .ok_or_else(|| SuiError::ObjectDeserializationError { + error: "Cannot extract [name] field from sui::dynamic_object_field::Wrapper." + .to_string(), + })?; + + (name.to_string(), DynamicFieldType::DynamicObject, object_id) + } else { + ( + name.to_string(), + DynamicFieldType::DynamicField { + wrapped_object_id: object_id, + }, + object_id, + ) + }) + } +} + +fn extract_field_from_move_struct(move_struct: &MoveStruct, field_name: &str) -> Option { + match move_struct { + MoveStruct::WithTypes { fields, .. } => fields.iter().find_map(|(id, value)| { + if id.to_string() == field_name { + Some(value.clone()) + } else { + None + } + }), + _ => None, + } +} + +fn extract_object_id(value: &MoveValue) -> Option { + match value { + MoveValue::Struct(MoveStruct::WithTypes { type_, fields }) => { + if type_ == &ID::type_() { + match fields.first() { + Some((_, MoveValue::Address(addr))) => Some(ObjectID::from(*addr)), + _ => None, + } + } else { + for (_, value) in fields { + let id = extract_object_id(value); + if id.is_some() { + return id; + } + } + None + } + } + _ => None, + } +} + +pub fn is_dynamic_object(move_struct: &MoveStruct) -> bool { + match move_struct { + MoveStruct::WithTypes { type_, .. } => { + matches!(&type_.type_params[0], TypeTag::Struct(tag) if tag.address == SUI_FRAMEWORK_ADDRESS + && tag.module.as_str() == "dynamic_object_field" + && tag.name.as_str() == "Wrapper") + } + _ => false, + } +} diff --git a/crates/sui-types/src/error.rs b/crates/sui-types/src/error.rs index 7cdff5b0092e9..28fcd16cbdbf6 100644 --- a/crates/sui-types/src/error.rs +++ b/crates/sui-types/src/error.rs @@ -435,6 +435,8 @@ pub enum SuiError { // Errors returned by authority and client read API's #[error("Failure serializing object in the requested format: {:?}", error)] ObjectSerializationError { error: String }, + #[error("Failure deserializing object in the requested format: {:?}", error)] + ObjectDeserializationError { error: String }, #[error("Event store component is not active on this node")] NoEventStore, diff --git a/crates/sui-types/src/lib.rs b/crates/sui-types/src/lib.rs index b2320818177a3..2904e782af719 100644 --- a/crates/sui-types/src/lib.rs +++ b/crates/sui-types/src/lib.rs @@ -29,6 +29,7 @@ pub mod coin; pub mod collection_types; pub mod committee; pub mod crypto; +pub mod dynamic_field; pub mod event; pub mod gas; pub mod gas_coin; From 04e73b7984381f0204a734da12b694683d764627 Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Nov 2022 20:36:31 +0000 Subject: [PATCH 04/13] added get_dynamic_field_object method --- crates/sui-core/src/lib.rs | 2 -- crates/sui-json-rpc/src/api.rs | 14 ++++++++++++-- crates/sui-json-rpc/src/lib.rs | 2 ++ crates/sui-json-rpc/src/read_api.rs | 21 +++++++++++++++++++-- crates/sui-types/src/dynamic_field.rs | 4 ++-- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/crates/sui-core/src/lib.rs b/crates/sui-core/src/lib.rs index 0354f34855530..4db074ad82fdd 100644 --- a/crates/sui-core/src/lib.rs +++ b/crates/sui-core/src/lib.rs @@ -1,5 +1,3 @@ -extern crate core; - // Copyright (c) 2021, Facebook, Inc. and its affiliates // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index ae346e1d7c6db..968d152c460cc 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -93,8 +93,8 @@ pub trait RpcReadApi { #[method(name = "getDynamicFields")] async fn get_dynamic_fields( &self, - /// the ID of the owner object - object_id: ObjectID, + /// the ID of the parent object + parent_object_id: ObjectID, ) -> RpcResult; /// Return the total number of transactions known to the server. @@ -134,6 +134,16 @@ pub trait RpcReadApi { /// the ID of the queried object object_id: ObjectID, ) -> RpcResult; + + /// Return the dynamic field object information for a specified object + #[method(name = "getDynamicFieldObject")] + async fn get_dynamic_field_object( + &self, + /// the ID of the queried parent object + parent_object_id: ObjectID, + /// the Name of the dynamic field + name: String, + ) -> RpcResult; } #[open_rpc(namespace = "sui", tag = "Full Node API")] diff --git a/crates/sui-json-rpc/src/lib.rs b/crates/sui-json-rpc/src/lib.rs index cb772cce913d7..e4b77e84acb92 100644 --- a/crates/sui-json-rpc/src/lib.rs +++ b/crates/sui-json-rpc/src/lib.rs @@ -1,6 +1,8 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +extern crate core; + use std::env; use std::net::SocketAddr; use std::str::FromStr; diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index fece0411bf927..d3551351002bc 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -76,10 +76,10 @@ impl RpcReadApiServer for ReadApi { .collect()) } - async fn get_dynamic_fields(&self, object_id: ObjectID) -> RpcResult { + async fn get_dynamic_fields(&self, parent_object_id: ObjectID) -> RpcResult { let data = self .state - .get_dynamic_fields(object_id) + .get_dynamic_fields(parent_object_id) .map_err(|e| anyhow!("{e}"))?; Ok(DynamicFieldPage { @@ -100,6 +100,23 @@ impl RpcReadApiServer for ReadApi { .try_into()?) } + async fn get_dynamic_field_object( + &self, + parent_object_id: ObjectID, + name: String, + ) -> RpcResult { + let data = self + .state + .get_dynamic_fields(parent_object_id) + .map_err(|e| anyhow!("{e}"))?; + + let df = data.iter().find(|info| info.name == name).ok_or_else(|| { + anyhow!("Cannot find dynamic field [{name}] for object [{parent_object_id}].") + })?; + + self.get_object(df.object_id).await + } + async fn get_total_transaction_number(&self) -> RpcResult { Ok(self.state.get_total_transaction_number()?) } diff --git a/crates/sui-types/src/dynamic_field.rs b/crates/sui-types/src/dynamic_field.rs index b20cc61bf5381..5696558e78717 100644 --- a/crates/sui-types/src/dynamic_field.rs +++ b/crates/sui-types/src/dynamic_field.rs @@ -42,13 +42,13 @@ impl DynamicFieldInfo { pub fn parse_move_object( move_struct: &MoveStruct, ) -> SuiResult<(String, DynamicFieldType, ObjectID)> { - let name = extract_field_from_move_struct(&move_struct, "name").ok_or_else(|| { + let name = extract_field_from_move_struct(move_struct, "name").ok_or_else(|| { SuiError::ObjectDeserializationError { error: "Cannot extract [name] field from sui::dynamic_field::Field".to_string(), } })?; - let value = extract_field_from_move_struct(&move_struct, "value").ok_or_else(|| { + let value = extract_field_from_move_struct(move_struct, "value").ok_or_else(|| { SuiError::ObjectDeserializationError { error: "Cannot extract [value] field from sui::dynamic_field::Field".to_string(), } From 5cdaa55ab80c7b4c0f792e5070ed266fc70f161d Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Nov 2022 21:52:31 +0000 Subject: [PATCH 05/13] handle none optional value in the Field object --- crates/sui-core/src/authority/authority_store.rs | 9 +++++---- crates/sui-types/src/dynamic_field.rs | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index eecae0fa81067..7da462b7ee848 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -635,9 +635,7 @@ impl AuthorityStore { uncommitted_objects: &BTreeMap, ) -> SuiResult> { // Skip if not a move object - let move_object = if let Some(move_object) = o.data.try_as_move() { - move_object - } else { + let Some(move_object) = o.data.try_as_move() else { return Ok(None); }; // We only index dynamic field objects @@ -647,7 +645,10 @@ impl AuthorityStore { let move_struct = move_object.to_move_struct_with_resolver(ObjectFormatOptions::default(), &self)?; - let (name, type_, object_id) = DynamicFieldInfo::parse_move_object(&move_struct)?; + let Some((name, type_, object_id)) = DynamicFieldInfo::parse_move_object(&move_struct)? else{ + return Ok(None) + }; + Ok(Some(match type_ { DynamicFieldType::DynamicObject => { // Find the actual object from storage using the object id obtained from the wrapper. diff --git a/crates/sui-types/src/dynamic_field.rs b/crates/sui-types/src/dynamic_field.rs index 5696558e78717..40995f0501b66 100644 --- a/crates/sui-types/src/dynamic_field.rs +++ b/crates/sui-types/src/dynamic_field.rs @@ -41,7 +41,7 @@ impl DynamicFieldInfo { pub fn parse_move_object( move_struct: &MoveStruct, - ) -> SuiResult<(String, DynamicFieldType, ObjectID)> { + ) -> SuiResult> { let name = extract_field_from_move_struct(move_struct, "name").ok_or_else(|| { SuiError::ObjectDeserializationError { error: "Cannot extract [name] field from sui::dynamic_field::Field".to_string(), @@ -55,16 +55,16 @@ impl DynamicFieldInfo { })?; // Extract value from value option - let value = match &value { + let Some(value) = (match &value { MoveValue::Struct(MoveStruct::WithTypes { type_: _, fields }) => match fields.first() { Some((_, MoveValue::Vector(v))) => v.first().cloned(), _ => None, }, _ => None, - } - .ok_or_else(|| SuiError::ObjectDeserializationError { - error: "Cannot extract optional value".to_string(), - })?; + }) else { + // Skip processing if Field's value is not set. + return Ok(None) + }; let object_id = extract_object_id(&value).ok_or_else(|| SuiError::ObjectDeserializationError { @@ -74,7 +74,7 @@ impl DynamicFieldInfo { ), })?; - Ok(if is_dynamic_object(move_struct) { + Ok(Some(if is_dynamic_object(move_struct) { let name = match name { MoveValue::Struct(s) => extract_field_from_move_struct(&s, "name"), _ => None, @@ -93,7 +93,7 @@ impl DynamicFieldInfo { }, object_id, ) - }) + })) } } From 478037f81eea58f24ec25c6ae80a2489cbc01444 Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 23 Nov 2022 23:06:20 +0000 Subject: [PATCH 06/13] clean up, revert unnecessary import changes --- .../sui-core/src/authority/authority_store.rs | 13 ++++------- .../src/authority/authority_store_tables.rs | 6 ++++- .../src/unit_tests/authority_tests.rs | 23 ++++++++++--------- crates/sui-json-rpc/src/lib.rs | 2 -- crates/sui-json-rpc/src/read_api.rs | 22 ------------------ crates/sui-types/src/base_types.rs | 6 ++--- crates/sui-types/src/object.rs | 2 +- 7 files changed, 25 insertions(+), 49 deletions(-) diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index 7da462b7ee848..e0259cb79eb44 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -2,15 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::iter; use std::path::Path; use std::sync::Arc; use std::{fmt::Debug, path::PathBuf}; -use super::{authority_store_tables::AuthorityPerpetualTables, *}; -use crate::authority::authority_per_epoch_store::{ - AuthorityPerEpochStore, ExecutionIndicesWithHash, -}; use arc_swap::ArcSwap; use move_binary_format::CompiledModule; use move_bytecode_utils::module_cache::GetModule; @@ -22,11 +19,6 @@ use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tracing::{debug, info, trace}; use narwhal_executor::ExecutionIndices; -use std::collections::BTreeSet; -use std::iter; -use std::path::Path; -use std::sync::Arc; -use std::{fmt::Debug, path::PathBuf}; use sui_storage::{ lock_service::ObjectLockStatus, mutex_table::{LockGuard, MutexTable}, @@ -42,6 +34,9 @@ use sui_types::{batch::TxSequenceNumber, object::PACKAGE_VERSION}; use typed_store::rocks::DBBatch; use typed_store::traits::Map; +use crate::authority::authority_per_epoch_store::{ + AuthorityPerEpochStore, ExecutionIndicesWithHash, +}; use crate::authority::authority_store_tables::ExecutionIndicesWithHash; use super::{ diff --git a/crates/sui-core/src/authority/authority_store_tables.rs b/crates/sui-core/src/authority/authority_store_tables.rs index bce5f0a21faea..fd6226fd8da6a 100644 --- a/crates/sui-core/src/authority/authority_store_tables.rs +++ b/crates/sui-core/src/authority/authority_store_tables.rs @@ -31,12 +31,16 @@ pub struct AuthorityPerpetualTables { #[default_options_override_fn = "objects_table_default_config"] pub(crate) objects: DBMap, - /// This is a an index of object references to currently existing objects, indexed by the + /// This is an index of object references to currently existing objects, indexed by the /// composite key of the SuiAddress of their owner and the object ID of the object. /// This composite index allows an efficient iterator to list all objected currently owned /// by a specific user, and their object reference. pub(crate) owner_index: DBMap<(SuiAddress, ObjectID), ObjectInfo>, + /// This is an index of object references to currently existing objects, indexed by the + /// composite key of the object ID of their parent and the object ID of the object. + /// This composite index allows an efficient iterator to list all objected currently owned + /// by a specific object, and their object reference. pub(crate) dynamic_field_index: DBMap<(ObjectID, ObjectID), DynamicFieldInfo>, /// This is a map between the transaction digest and the corresponding certificate for all diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index 5cfa8c1a7c45c..38393a38d0f3b 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -3014,19 +3014,17 @@ pub async fn init_state_with_ids_and_object_basics< >( objects: I, ) -> (Arc, ObjectRef) { - let state = init_state_with_ids(objects).await; - let pkg = build_package("src/unit_tests/data/object_basics"); - let pkg_ref = pkg.compute_object_reference(); - state.insert_genesis_object(pkg).await; - (state, pkg_ref) -} - -#[cfg(test)] -pub fn build_package(package_path: &str) -> Object { use sui_framework_build::compiled_package::BuildConfig; + let state = init_state().await; + for (address, object_id) in objects { + let obj = Object::with_id_owner_for_testing(object_id, address); + state.insert_genesis_object(obj).await; + } + + // add object_basics package object to genesis, since lots of test use it let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push(package_path); + path.push("src/unit_tests/data/object_basics"); let modules = BuildConfig::default() .build(path) .unwrap() @@ -3034,7 +3032,10 @@ pub fn build_package(package_path: &str) -> Object { .into_iter() .cloned() .collect(); - Object::new_package(modules, TransactionDigest::genesis().unwrap()) + let pkg = Object::new_package(modules, TransactionDigest::genesis().unwrap()); + let pkg_ref = pkg.compute_object_reference(); + state.insert_genesis_object(pkg).await; + (state, pkg_ref) } #[cfg(test)] diff --git a/crates/sui-json-rpc/src/lib.rs b/crates/sui-json-rpc/src/lib.rs index e4b77e84acb92..cb772cce913d7 100644 --- a/crates/sui-json-rpc/src/lib.rs +++ b/crates/sui-json-rpc/src/lib.rs @@ -1,8 +1,6 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -extern crate core; - use std::env; use std::net::SocketAddr; use std::str::FromStr; diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index d3551351002bc..46e39190aaba4 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -397,25 +397,3 @@ pub async fn get_move_modules_by_package( _ => Err(anyhow!("Package object does not exist with ID {}", package)), }?) } - -/* -#[test] -fn test() { - let tag = "0x2::dynamic_field::Field<0x2::dynamic_object_field::Wrapper<0x2::typed_id::TypedID<0xdf496988c6d4571684d9f4edcaec5a1cf889fa2c::geniteam::Farm>>, 0x2::object::ID>"; - let tag2 = "0x2::dynamic_field::Field<0x2::object::ID, 0xdf496988c6d4571684d9f4edcaec5a1cf889fa2c::marketplace::Listing<0x2::devnet_nft::DevNetNFT, 0x2::sui::SUI>>"; - - let tag = parse_sui_struct_tag(tag).unwrap(); - let tag2 = parse_sui_struct_tag(tag2).unwrap(); - - for tag in [tag, tag2] { - if is_dynamic_field(&tag) { - if matches!(&tag.type_params[0], TypeTag::Struct(tag) if is_dynamic_field_wrapper(&tag)) - { - println!("1") - } else { - println!("2") - } - } - } -} -*/ diff --git a/crates/sui-types/src/base_types.rs b/crates/sui-types/src/base_types.rs index 5d0a59d017b24..682c16055159a 100644 --- a/crates/sui-types/src/base_types.rs +++ b/crates/sui-types/src/base_types.rs @@ -5,10 +5,10 @@ use anyhow::anyhow; use curve25519_dalek::ristretto::RistrettoPoint; use fastcrypto::encoding::decode_bytes_hex; -use fastcrypto::encoding::{Base64, Encoding, Hex}; use move_core_types::account_address::AccountAddress; use move_core_types::ident_str; use move_core_types::identifier::IdentStr; +use move_core_types::language_storage::StructTag; use opentelemetry::{global, Context}; use rand::Rng; use schemars::JsonSchema; @@ -34,10 +34,10 @@ use crate::error::SuiError; use crate::gas_coin::GasCoin; use crate::object::{Object, Owner}; use crate::sui_serde::Readable; -use fastcrypto::encoding::{Base58, Base64, Encoding, Hex}; +use crate::waypoint::IntoPoint; use crate::SUI_FRAMEWORK_ADDRESS; +use fastcrypto::encoding::{Base58, Base64, Encoding, Hex}; use fastcrypto::hash::{HashFunction, Sha3_256}; -use move_core_types::language_storage::StructTag; #[cfg(test)] #[path = "unit_tests/base_types_tests.rs"] diff --git a/crates/sui-types/src/object.rs b/crates/sui-types/src/object.rs index a8cf4d50baa5f..05159ea772a91 100644 --- a/crates/sui-types/src/object.rs +++ b/crates/sui-types/src/object.rs @@ -57,7 +57,7 @@ pub struct ObjectFormatOptions { /// `{ "fields": { "f": 20, "g": { "fields" { "h": true }, "type": "0x0::MyModule::MyNestedType" }, "type": "0x0::MyModule::MyType" }` /// If false, include field names only; e.g.: /// `{ "f": 20, "g": { "h": true } }` - pub include_types: bool, + include_types: bool, } impl MoveObject { From 55f2b6a1f636336e55fd7db106b8b58408dd76bf Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 24 Nov 2022 10:56:50 +0000 Subject: [PATCH 07/13] add pagination cursor and limit to dynamic field --- crates/sui-core/src/authority.rs | 17 ++++++++- .../sui-core/src/authority/authority_store.rs | 31 ++++++++++++--- .../src/authority/authority_store_tables.rs | 4 +- .../src/unit_tests/authority_tests.rs | 8 +++- crates/sui-json-rpc-types/src/lib.rs | 2 +- crates/sui-json-rpc/src/api.rs | 22 ++++++----- crates/sui-json-rpc/src/event_api.rs | 2 +- crates/sui-json-rpc/src/read_api.rs | 38 ++++++++++--------- 8 files changed, 84 insertions(+), 40 deletions(-) diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index 5fea7030cdc92..56b2809397670 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -1867,8 +1867,21 @@ impl AuthorityState { self.database.get_owner_objects(owner) } - pub fn get_dynamic_fields(&self, owner: ObjectID) -> SuiResult> { - self.database.get_dynamic_fields(owner) + pub fn get_dynamic_fields( + &self, + owner: ObjectID, + cursor: Option, + limit: usize, + ) -> SuiResult> { + self.database.get_dynamic_fields(owner, cursor, limit) + } + + pub fn get_dynamic_field_object_id( + &self, + owner: ObjectID, + name: &str, + ) -> SuiResult> { + self.database.get_dynamic_field_object_id(owner, name) } pub fn get_total_transaction_number(&self) -> Result { diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index e0259cb79eb44..e991444aacb58 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -1,8 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::iter; use std::path::Path; use std::sync::Arc; @@ -242,20 +241,42 @@ impl AuthorityStore { .map(|(_, object_info)| object_info)) } - // Methods to read the store - pub fn get_dynamic_fields(&self, object: ObjectID) -> Result, SuiError> { + pub fn get_dynamic_fields( + &self, + object: ObjectID, + cursor: Option, + limit: usize, + ) -> Result, SuiError> { debug!(?object, "get_dynamic_fields"); + let cursor = cursor.unwrap_or(ObjectID::ZERO); Ok(self .perpetual_tables .dynamic_field_index .iter() // The object id 0 is the smallest possible - .skip_to(&(object, ObjectID::ZERO))? + .skip_to(&(object, cursor))? .take_while(|((object_owner, _), _)| (object_owner == &object)) .map(|(_, object_info)| object_info) + .take(limit) .collect()) } + pub fn get_dynamic_field_object_id( + &self, + object: ObjectID, + name: &str, + ) -> Result, SuiError> { + debug!(?object, "get_dynamic_field_object_id"); + Ok(self + .perpetual_tables + .dynamic_field_index + .iter() + // The object id 0 is the smallest possible + .skip_to(&(object, ObjectID::ZERO))? + .find(|((object_owner, _), info)| (object_owner == &object && info.name == name)) + .map(|(_, object_info)| object_info.object_id)) + } + pub fn get_object_by_key( &self, object_id: &ObjectID, diff --git a/crates/sui-core/src/authority/authority_store_tables.rs b/crates/sui-core/src/authority/authority_store_tables.rs index fd6226fd8da6a..ed6a162d54c38 100644 --- a/crates/sui-core/src/authority/authority_store_tables.rs +++ b/crates/sui-core/src/authority/authority_store_tables.rs @@ -37,8 +37,8 @@ pub struct AuthorityPerpetualTables { /// by a specific user, and their object reference. pub(crate) owner_index: DBMap<(SuiAddress, ObjectID), ObjectInfo>, - /// This is an index of object references to currently existing objects, indexed by the - /// composite key of the object ID of their parent and the object ID of the object. + /// This is an index of object references to currently existing dynamic field object, indexed by the + /// composite key of the object ID of their parent and the object ID of the dynamic field object. /// This composite index allows an efficient iterator to list all objected currently owned /// by a specific object, and their object reference. pub(crate) dynamic_field_index: DBMap<(ObjectID, ObjectID), DynamicFieldInfo>, diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index 38393a38d0f3b..a0f130b066469 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -2684,7 +2684,9 @@ async fn test_store_get_dynamic_object() { assert!(add_effects.status.is_ok()); assert_eq!(add_effects.created.len(), 1); - let fields = authority_state.get_dynamic_fields(outer_v0.0).unwrap(); + let fields = authority_state + .get_dynamic_fields(outer_v0.0, None, usize::MAX) + .unwrap(); assert_eq!(fields.len(), 1); assert_eq!(fields[0].type_, DynamicFieldType::DynamicObject); } @@ -2755,7 +2757,9 @@ async fn test_store_get_dynamic_field() { assert!(add_effects.status.is_ok()); assert_eq!(add_effects.created.len(), 1); - let fields = authority_state.get_dynamic_fields(outer_v0.0).unwrap(); + let fields = authority_state + .get_dynamic_fields(outer_v0.0, None, usize::MAX) + .unwrap(); assert_eq!(fields.len(), 1); assert!( matches!(fields[0].type_, DynamicFieldType::DynamicField {wrapped_object_id} if wrapped_object_id == inner_v0.0) diff --git a/crates/sui-json-rpc-types/src/lib.rs b/crates/sui-json-rpc-types/src/lib.rs index 855e8e97e324c..0f4b041d461ab 100644 --- a/crates/sui-json-rpc-types/src/lib.rs +++ b/crates/sui-json-rpc-types/src/lib.rs @@ -66,7 +66,7 @@ pub type SuiMoveTypeParameterIndex = u16; pub type TransactionsPage = Page; pub type EventPage = Page; pub type CoinPage = Page; -pub type DynamicFieldPage = Page; +pub type DynamicFieldPage = Page; #[derive(Serialize, Deserialize, Debug, JsonSchema)] #[serde(rename_all = "camelCase")] diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index 968d152c460cc..48e52fc700d06 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -1,13 +1,13 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::collections::BTreeMap; + use anyhow::anyhow; +use fastcrypto::encoding::Base64; use jsonrpsee::core::RpcResult; use jsonrpsee_proc_macros::rpc; -use std::collections::BTreeMap; -use sui_types::sui_system_state::SuiSystemState; -use fastcrypto::encoding::Base64; use sui_json::SuiJsonValue; use sui_json_rpc_types::{ Balance, CoinPage, EventPage, GetObjectDataResponse, GetPastObjectDataResponse, @@ -27,6 +27,7 @@ use sui_types::event::EventID; use sui_types::messages::CommitteeInfoResponse; use sui_types::messages::ExecuteTransactionRequestType; use sui_types::query::{EventQuery, TransactionQuery}; +use sui_types::sui_system_state::SuiSystemState; /// Maximum number of events returned in an event query. /// This is equivalent to EVENT_QUERY_MAX_LIMIT in `sui-storage` crate. @@ -95,6 +96,10 @@ pub trait RpcReadApi { &self, /// the ID of the parent object parent_object_id: ObjectID, + /// Optional paging cursor + cursor: Option, + /// Maximum item returned per page + limit: Option, ) -> RpcResult; /// Return the total number of transactions known to the server. @@ -534,14 +539,11 @@ pub trait TransactionExecutionApi { ) -> RpcResult; } -pub fn cap_page_limit(limit: Option) -> Result { - let limit = limit.unwrap_or(QUERY_MAX_RESULT_LIMIT); - if limit == 0 { - Err(anyhow!("Page result limit must be larger then 0."))?; - } - Ok(if limit > QUERY_MAX_RESULT_LIMIT { +pub fn cap_page_limit(limit: Option) -> usize { + let limit = limit.unwrap_or_default(); + if limit > QUERY_MAX_RESULT_LIMIT || limit == 0 { QUERY_MAX_RESULT_LIMIT } else { limit - }) + } } diff --git a/crates/sui-json-rpc/src/event_api.rs b/crates/sui-json-rpc/src/event_api.rs index 78d093a3afec1..0d71ebed754ca 100644 --- a/crates/sui-json-rpc/src/event_api.rs +++ b/crates/sui-json-rpc/src/event_api.rs @@ -103,7 +103,7 @@ impl EventReadApiServer for EventReadApiImpl { descending_order: Option, ) -> RpcResult { let descending = descending_order.unwrap_or_default(); - let limit = cap_page_limit(limit)?; + let limit = cap_page_limit(limit); // Retrieve 1 extra item for next cursor let mut data = self .state diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index 46e39190aaba4..80dbd758268a3 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -76,16 +76,22 @@ impl RpcReadApiServer for ReadApi { .collect()) } - async fn get_dynamic_fields(&self, parent_object_id: ObjectID) -> RpcResult { - let data = self + async fn get_dynamic_fields( + &self, + parent_object_id: ObjectID, + cursor: Option, + limit: Option, + ) -> RpcResult { + let limit = cap_page_limit(limit); + let mut data = self .state - .get_dynamic_fields(parent_object_id) + .get_dynamic_fields(parent_object_id, cursor, limit + 1) .map_err(|e| anyhow!("{e}"))?; - Ok(DynamicFieldPage { - data, - next_cursor: None, - }) + let next_cursor = data.get(limit).map(|info| info.object_id); + data.truncate(limit); + + Ok(DynamicFieldPage { data, next_cursor }) } async fn get_object(&self, object_id: ObjectID) -> RpcResult { @@ -105,16 +111,14 @@ impl RpcReadApiServer for ReadApi { parent_object_id: ObjectID, name: String, ) -> RpcResult { - let data = self + let id = self .state - .get_dynamic_fields(parent_object_id) - .map_err(|e| anyhow!("{e}"))?; - - let df = data.iter().find(|info| info.name == name).ok_or_else(|| { - anyhow!("Cannot find dynamic field [{name}] for object [{parent_object_id}].") - })?; - - self.get_object(df.object_id).await + .get_dynamic_field_object_id(parent_object_id, &name) + .map_err(|e| anyhow!("{e}"))? + .ok_or_else(|| { + anyhow!("Cannot find dynamic field [{name}] for object [{parent_object_id}].") + })?; + self.get_object(id).await } async fn get_total_transaction_number(&self) -> RpcResult { @@ -312,7 +316,7 @@ impl RpcFullNodeReadApiServer for FullNodeApi { limit: Option, descending_order: Option, ) -> RpcResult { - let limit = cap_page_limit(limit)?; + let limit = cap_page_limit(limit); let descending = descending_order.unwrap_or_default(); // Retrieve 1 extra item for next cursor From 317908fa0ee7408b21c0ae2160b74f893865222d Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 24 Nov 2022 11:39:19 +0000 Subject: [PATCH 08/13] added back sui_getObjectsOwnedByObject for backward compatibility --- .../sui-core/src/authority/authority_store.rs | 6 +--- .../src/unit_tests/authority_tests.rs | 2 +- crates/sui-json-rpc/src/api.rs | 8 +++++ crates/sui-json-rpc/src/read_api.rs | 33 +++++++++++++++++-- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index e991444aacb58..fd0be5156ed59 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -36,12 +36,8 @@ use typed_store::traits::Map; use crate::authority::authority_per_epoch_store::{ AuthorityPerEpochStore, ExecutionIndicesWithHash, }; -use crate::authority::authority_store_tables::ExecutionIndicesWithHash; -use super::{ - authority_store_tables::{AuthorityEpochTables, AuthorityPerpetualTables}, - *, -}; +use super::{authority_store_tables::AuthorityPerpetualTables, *}; pub type AuthorityStore = SuiDataStore; diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index a0f130b066469..5c6d249132bff 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -3036,7 +3036,7 @@ pub async fn init_state_with_ids_and_object_basics< .into_iter() .cloned() .collect(); - let pkg = Object::new_package(modules, TransactionDigest::genesis().unwrap()); + let pkg = Object::new_package(modules, TransactionDigest::genesis()).unwrap(); let pkg_ref = pkg.compute_object_reference(); state.insert_genesis_object(pkg).await; (state, pkg_ref) diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index 48e52fc700d06..882dc20d8c018 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -91,6 +91,14 @@ pub trait RpcReadApi { ) -> RpcResult>; /// Return the list of objects owned by an object. + #[method(name = "getObjectsOwnedByObject")] + async fn get_objects_owned_by_object( + &self, + /// the ID of the owner object + object_id: ObjectID, + ) -> RpcResult>; + + /// Return the list of dynamic field objects owned by an object. #[method(name = "getDynamicFields")] async fn get_dynamic_fields( &self, diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index 80dbd758268a3..ef6cb983530a3 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -7,7 +7,6 @@ use jsonrpsee::core::RpcResult; use move_binary_format::normalized::{Module as NormalizedModule, Type}; use move_core_types::identifier::Identifier; use move_core_types::language_storage::StructTag; -use signature::Signature; use std::collections::BTreeMap; use std::sync::Arc; use sui_types::intent::{Intent, IntentMessage}; @@ -76,6 +75,36 @@ impl RpcReadApiServer for ReadApi { .collect()) } + async fn get_objects_owned_by_object( + &self, + object_id: ObjectID, + ) -> RpcResult> { + let dynamic_fields: DynamicFieldPage = + self.get_dynamic_fields(object_id, None, None).await?; + + let mut object_info = vec![]; + for info in dynamic_fields.data { + // TODO: Remove this + // This is very expensive, it's only for backward compatibilities and should be removed asap. + let object = self + .state + .get_object_read(&info.object_id) + .await + .and_then(|read| read.into_object()) + .map_err(|e| anyhow!(e))?; + object_info.push(SuiObjectInfo { + object_id: object.id(), + version: object.version(), + digest: object.digest(), + // Package cannot be owned by object, safe to unwrap. + type_: format!("{}", object.type_().unwrap()), + owner: object.owner, + previous_transaction: object.previous_transaction, + }); + } + Ok(object_info) + } + async fn get_dynamic_fields( &self, parent_object_id: ObjectID, @@ -87,10 +116,8 @@ impl RpcReadApiServer for ReadApi { .state .get_dynamic_fields(parent_object_id, cursor, limit + 1) .map_err(|e| anyhow!("{e}"))?; - let next_cursor = data.get(limit).map(|info| info.object_id); data.truncate(limit); - Ok(DynamicFieldPage { data, next_cursor }) } From 0d6efee4eed6cd9e3be276a6803ece430c76447a Mon Sep 17 00:00:00 2001 From: Patrick Kuo Date: Wed, 30 Nov 2022 17:10:39 +0000 Subject: [PATCH 09/13] Update crates/sui-json-rpc/src/api.rs Co-authored-by: ronny-mysten <118224482+ronny-mysten@users.noreply.github.com> --- crates/sui-core/src/authority/authority_store.rs | 16 ++++------------ .../src/authority/authority_store_tables.rs | 2 +- .../src/unit_tests/move_integration_tests.rs | 1 + crates/sui-json-rpc/src/api.rs | 9 ++++++--- crates/sui/src/client_commands.rs | 2 +- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index fd0be5156ed59..e12481c9cea01 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -11,6 +11,7 @@ use arc_swap::ArcSwap; use move_binary_format::CompiledModule; use move_bytecode_utils::module_cache::GetModule; use once_cell::sync::OnceCell; +use parking_lot::RwLock; use rocksdb::Options; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -1200,17 +1201,8 @@ impl AuthorityStore { let modified_object_keys = effects .modified_at_versions .iter() - .map(|(r, _)| r) - .chain(effects.deleted.iter()) - .chain(effects.wrapped.iter()) - .map(|(id, version, _)| { - ObjectKey( - *id, - version - .decrement() - .expect("version revert should never fail"), - ) - }); + .map(|(id, version)| ObjectKey(*id, *version)); + let (old_objects_and_locks, old_dynamic_fields): (Vec<_>, Vec<_>) = self .perpetual_tables .objects @@ -1234,7 +1226,7 @@ impl AuthorityStore { }) .unzip(); - let (old_objects, old_locks): (Vec<_>, Vec<_>) = + let (old_modified_objects, old_locks): (Vec<_>, Vec<_>) = old_objects_and_locks.into_iter().flatten().unzip(); let old_dynamic_fields = old_dynamic_fields .into_iter() diff --git a/crates/sui-core/src/authority/authority_store_tables.rs b/crates/sui-core/src/authority/authority_store_tables.rs index ed6a162d54c38..f440bec9413f1 100644 --- a/crates/sui-core/src/authority/authority_store_tables.rs +++ b/crates/sui-core/src/authority/authority_store_tables.rs @@ -39,7 +39,7 @@ pub struct AuthorityPerpetualTables { /// This is an index of object references to currently existing dynamic field object, indexed by the /// composite key of the object ID of their parent and the object ID of the dynamic field object. - /// This composite index allows an efficient iterator to list all objected currently owned + /// This composite index allows an efficient iterator to list all objects currently owned /// by a specific object, and their object reference. pub(crate) dynamic_field_index: DBMap<(ObjectID, ObjectID), DynamicFieldInfo>, diff --git a/crates/sui-core/src/unit_tests/move_integration_tests.rs b/crates/sui-core/src/unit_tests/move_integration_tests.rs index fb3d096cf260f..3872b2f16a18d 100644 --- a/crates/sui-core/src/unit_tests/move_integration_tests.rs +++ b/crates/sui-core/src/unit_tests/move_integration_tests.rs @@ -23,6 +23,7 @@ use sui_types::{ use std::path::PathBuf; use std::{env, str::FromStr}; +use sui_types::object::Owner; const MAX_GAS: u64 = 10000; diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index 882dc20d8c018..1f22afcef8c7d 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -13,6 +13,9 @@ use sui_json_rpc_types::{ Balance, CoinPage, EventPage, GetObjectDataResponse, GetPastObjectDataResponse, GetRawObjectDataResponse, MoveFunctionArgType, RPCTransactionRequestParams, SuiCoinMetadata, SuiEventEnvelope, SuiEventFilter, SuiExecuteTransactionResponse, SuiMoveNormalizedFunction, + DynamicFieldPage, EventPage, GetObjectDataResponse, GetPastObjectDataResponse, + GetRawObjectDataResponse, MoveFunctionArgType, RPCTransactionRequestParams, SuiCoinMetadata, + SuiEventEnvelope, SuiEventFilter, SuiExecuteTransactionResponse, SuiMoveNormalizedFunction, SuiMoveNormalizedModule, SuiMoveNormalizedStruct, SuiObjectInfo, SuiTransactionAuthSignersResponse, SuiTransactionEffects, SuiTransactionFilter, SuiTransactionResponse, SuiTypeTag, TransactionBytes, TransactionsPage, @@ -102,7 +105,7 @@ pub trait RpcReadApi { #[method(name = "getDynamicFields")] async fn get_dynamic_fields( &self, - /// the ID of the parent object + /// The ID of the parent object parent_object_id: ObjectID, /// Optional paging cursor cursor: Option, @@ -152,9 +155,9 @@ pub trait RpcReadApi { #[method(name = "getDynamicFieldObject")] async fn get_dynamic_field_object( &self, - /// the ID of the queried parent object + /// The ID of the queried parent object parent_object_id: ObjectID, - /// the Name of the dynamic field + /// The Name of the dynamic field name: String, ) -> RpcResult; } diff --git a/crates/sui/src/client_commands.rs b/crates/sui/src/client_commands.rs index c2ad326d56db4..8c7e1a4e6573b 100644 --- a/crates/sui/src/client_commands.rs +++ b/crates/sui/src/client_commands.rs @@ -682,7 +682,7 @@ impl SuiClientCommands { SuiClientCommands::Objects { address } => { let address = address.unwrap_or(context.active_address()?); let client = context.get_client().await?; - let mut address_object = client + let address_object = client .read_api() .get_objects_owned_by_address(address) .await?; From 1773095958775998fb7e22a30232841da33d8625 Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 7 Dec 2022 16:20:34 +0000 Subject: [PATCH 10/13] move owner_index and dynamic field index to index store. --- crates/sui-core/src/authority.rs | 199 ++++++++++- .../sui-core/src/authority/authority_store.rs | 325 +++--------------- .../src/authority/authority_store_tables.rs | 27 +- .../src/unit_tests/authority_tests.rs | 13 +- crates/sui-json-rpc/src/read_api.rs | 12 +- crates/sui-storage/src/indexes.rs | 109 +++++- crates/sui-types/src/error.rs | 3 + 7 files changed, 380 insertions(+), 308 deletions(-) diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index 56b2809397670..32d662eca3967 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -31,10 +31,10 @@ use tap::TapFallible; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc::unbounded_channel; use tracing::{debug, error, instrument, warn, Instrument}; -use typed_store::Map; pub use authority_notify_read::EffectsNotifyRead; pub use authority_store::{AuthorityStore, ResolverWrapper, UpdateType}; +use mysten_metrics::monitored_scope; use narwhal_config::{ Committee as ConsensusCommittee, WorkerCache as ConsensusWorkerCache, WorkerId as ConsensusWorkerId, @@ -47,6 +47,7 @@ use sui_json_rpc_types::{ SuiTransactionEffects, }; use sui_simulator::nondeterministic; +use sui_storage::indexes::ObjectIndexChanges; use sui_storage::write_ahead_log::WriteAheadLog; use sui_storage::{ event_store::{EventStore, EventStoreType, StoredEvent}, @@ -56,10 +57,13 @@ use sui_storage::{ }; use sui_types::committee::EpochId; use sui_types::crypto::{AuthorityKeyPair, NetworkKeyPair}; +use sui_types::dynamic_field::{DynamicFieldInfo, DynamicFieldType}; use sui_types::event::{Event, EventID}; +use sui_types::gas::GasCostSummary; use sui_types::messages_checkpoint::{CheckpointRequest, CheckpointResponse}; -use sui_types::object::PastObjectRead; +use sui_types::object::{Owner, PastObjectRead}; use sui_types::query::{EventQuery, TransactionQuery}; +use sui_types::storage::WriteKind; use sui_types::sui_system_state::SuiSystemState; use sui_types::temporary_store::InnerTemporaryStore; pub use sui_types::temporary_store::TemporaryStore; @@ -75,6 +79,7 @@ use sui_types::{ storage::{BackingPackageStore, DeleteKind}, MOVE_STDLIB_ADDRESS, SUI_FRAMEWORK_ADDRESS, SUI_SYSTEM_STATE_OBJECT_ID, }; +use typed_store::Map; use crate::authority::authority_notifier::TransactionNotifierTicket; use crate::authority::authority_notify_read::NotifyRead; @@ -98,10 +103,6 @@ use crate::{ transaction_manager::TransactionManager, transaction_streamer::TransactionStreamer, }; -use narwhal_types::ConsensusOutput; -use sui_types::dynamic_field::DynamicFieldInfo; - -use sui_types::gas::GasCostSummary; use self::authority_store::ObjectKey; @@ -1256,6 +1257,8 @@ impl AuthorityState { effects: &SignedTransactionEffects, timestamp_ms: u64, ) -> SuiResult { + let changes = self.process_object_index(effects)?; + indexes.index_tx( cert.sender_address(), cert.data() @@ -1274,12 +1277,162 @@ impl AuthorityState { .move_calls() .iter() .map(|mc| (mc.package.0, mc.module.clone(), mc.function.clone())), + changes, seq, digest, timestamp_ms, ) } + fn process_object_index( + &self, + effects: &SignedTransactionEffects, + ) -> Result { + let mut deleted_owners = vec![]; + let mut deleted_dynamic_fields = vec![]; + for (id, version, _) in &effects.deleted { + match self.get_owner_at_version(id, *version)? { + Owner::AddressOwner(addr) => deleted_owners.push((addr, *id)), + Owner::ObjectOwner(object_id) => { + deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) + } + _ => {} + } + } + + let mut new_owners = vec![]; + let mut new_dynamic_fields = vec![]; + + let modified_at_version = effects + .modified_at_versions + .iter() + .cloned() + .collect::>(); + + for (oref, owner, kind) in effects.all_mutated() { + let id = &oref.0; + // For mutated objects, retrieve old owner and delete old index if there is a owner change. + if let WriteKind::Mutate = kind { + let Some(old_version) = modified_at_version.get(id) else{ + error!("Error processing object owner index for tx [{}], cannot find modified at version for mutated object [{id}].", effects.transaction_digest); + continue; + }; + let Some(old_object) = self.database.get_object_by_key(id, *old_version)? else { + error!("Error processing object owner index for tx [{}], cannot find object [{id}] at version [{old_version}].", effects.transaction_digest); + continue; + }; + if &old_object.owner != owner { + match old_object.owner { + Owner::AddressOwner(addr) => { + deleted_owners.push((addr, *id)); + } + Owner::ObjectOwner(object_id) => { + deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) + } + _ => {} + } + } else { + // Owner is the same, nothing to update. + continue; + } + } + + match owner { + Owner::AddressOwner(addr) => { + let Some(o) = self.database.get_object_by_key(id, oref.1)? else{ + continue; + }; + + let type_ = o + .type_() + .map(|type_| ObjectType::Struct(type_.clone())) + .unwrap_or(ObjectType::Package); + + new_owners.push(( + (*addr, *id), + ObjectInfo { + object_id: *id, + version: oref.1, + digest: oref.2, + type_, + owner: *owner, + previous_transaction: effects.transaction_digest, + }, + )); + } + Owner::ObjectOwner(owner) => { + let Some(df_info) = self.try_create_dynamic_field_info(oref)? else{ + continue; + }; + new_dynamic_fields.push(((ObjectID::from(*owner), *id), df_info)) + } + _ => {} + } + } + + Ok(ObjectIndexChanges { + deleted_owners, + deleted_dynamic_fields, + new_owners, + new_dynamic_fields, + }) + } + + fn try_create_dynamic_field_info( + &self, + oref: &ObjectRef, + ) -> SuiResult> { + // Skip if not a move object + let Some(move_object) = self + .database.get_object_by_key(&oref.0, oref.1)?.and_then(|o| o.data.try_as_move().cloned()) else { + return Ok(None); + }; + // We only index dynamic field objects + if !DynamicFieldInfo::is_dynamic_field(&move_object.type_) { + return Ok(None); + } + let move_struct = move_object.to_move_struct_with_resolver( + ObjectFormatOptions::default(), + self.module_cache.as_ref(), + )?; + + let Some((name, type_, object_id)) = DynamicFieldInfo::parse_move_object(&move_struct)? else{ + return Ok(None) + }; + + Ok(Some(match type_ { + DynamicFieldType::DynamicObject => { + // Find the actual object from storage using the object id obtained from the wrapper. + let Some(object) = self.database.find_object_lt_or_eq_version(object_id, oref.1) else{ + return Err(SuiError::ObjectNotFound { + object_id, + version: Some(oref.1), + }) + }; + let version = object.version(); + let digest = object.digest(); + let object_type = object.data.type_().unwrap(); + + DynamicFieldInfo { + name, + type_, + object_type: object_type.to_string(), + object_id, + version, + digest, + } + } + DynamicFieldType::DynamicField { .. } => DynamicFieldInfo { + name, + type_, + object_type: move_object.type_.type_params[1].to_string(), + object_id: oref.0, + version: oref.1, + digest: oref.2, + }, + })) + } + #[instrument(level = "debug", skip_all, fields(seq=?seq, tx_digest=?digest), err)] async fn post_process_one_tx( &self, @@ -1863,8 +2016,26 @@ impl AuthorityState { } } + fn get_owner_at_version( + &self, + object_id: &ObjectID, + version: SequenceNumber, + ) -> Result { + self.database + .get_object_by_key(object_id, version)? + .ok_or(SuiError::ObjectNotFound { + object_id: *object_id, + version: Some(version), + }) + .map(|o| o.owner) + } + pub fn get_owner_objects(&self, owner: SuiAddress) -> SuiResult> { - self.database.get_owner_objects(owner) + if let Some(indexes) = &self.indexes { + indexes.get_owner_objects(owner) + } else { + Err(SuiError::IndexStoreNotAvailable) + } } pub fn get_dynamic_fields( @@ -1873,7 +2044,11 @@ impl AuthorityState { cursor: Option, limit: usize, ) -> SuiResult> { - self.database.get_dynamic_fields(owner, cursor, limit) + if let Some(indexes) = &self.indexes { + indexes.get_dynamic_fields(owner, cursor, limit) + } else { + Err(SuiError::IndexStoreNotAvailable) + } } pub fn get_dynamic_field_object_id( @@ -1881,7 +2056,11 @@ impl AuthorityState { owner: ObjectID, name: &str, ) -> SuiResult> { - self.database.get_dynamic_field_object_id(owner, name) + if let Some(indexes) = &self.indexes { + indexes.get_dynamic_field_object_id(owner, name) + } else { + Err(SuiError::IndexStoreNotAvailable) + } } pub fn get_total_transaction_number(&self) -> Result { @@ -2144,7 +2323,7 @@ impl AuthorityState { fn make_account_info(&self, account: SuiAddress) -> Result { self.database - .get_owner_objects(account) + .get_owner_objects(Owner::AddressOwner(account)) .map(|object_ids| AccountInfoResponse { object_ids: object_ids.into_iter().map(|id| id.into()).collect(), owner: account, diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index e12481c9cea01..28143c1d53613 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -8,10 +8,7 @@ use std::sync::Arc; use std::{fmt::Debug, path::PathBuf}; use arc_swap::ArcSwap; -use move_binary_format::CompiledModule; -use move_bytecode_utils::module_cache::GetModule; use once_cell::sync::OnceCell; -use parking_lot::RwLock; use rocksdb::Options; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -25,7 +22,6 @@ use sui_storage::{ LockService, }; use sui_types::crypto::AuthoritySignInfo; -use sui_types::dynamic_field::{DynamicFieldInfo, DynamicFieldType}; use sui_types::message_envelope::VerifiedEnvelope; use sui_types::object::Owner; use sui_types::storage::ChildObjectResolver; @@ -67,8 +63,6 @@ pub struct AuthorityStore { // Implementation detail to support notify_read_effects(). pub(crate) effects_notify_read: NotifyRead, - - module_cache: RwLock>>, } impl AuthorityStore { @@ -132,7 +126,6 @@ impl AuthorityStore { path: path.into(), db_options, effects_notify_read: NotifyRead::new(), - module_cache: Default::default(), }; // Only initialize an empty database. if store @@ -217,7 +210,7 @@ impl AuthorityStore { } // Methods to read the store - pub fn get_owner_objects(&self, owner: SuiAddress) -> Result, SuiError> { + pub fn get_owner_objects(&self, owner: Owner) -> Result, SuiError> { debug!(?owner, "get_owner_objects"); Ok(self.get_owner_objects_iterator(owner)?.collect()) } @@ -238,42 +231,6 @@ impl AuthorityStore { .map(|(_, object_info)| object_info)) } - pub fn get_dynamic_fields( - &self, - object: ObjectID, - cursor: Option, - limit: usize, - ) -> Result, SuiError> { - debug!(?object, "get_dynamic_fields"); - let cursor = cursor.unwrap_or(ObjectID::ZERO); - Ok(self - .perpetual_tables - .dynamic_field_index - .iter() - // The object id 0 is the smallest possible - .skip_to(&(object, cursor))? - .take_while(|((object_owner, _), _)| (object_owner == &object)) - .map(|(_, object_info)| object_info) - .take(limit) - .collect()) - } - - pub fn get_dynamic_field_object_id( - &self, - object: ObjectID, - name: &str, - ) -> Result, SuiError> { - debug!(?object, "get_dynamic_field_object_id"); - Ok(self - .perpetual_tables - .dynamic_field_index - .iter() - // The object id 0 is the smallest possible - .skip_to(&(object, ObjectID::ZERO))? - .find(|((object_owner, _), info)| (object_owner == &object && info.name == name)) - .map(|(_, object_info)| object_info.object_id)) - } - pub fn get_object_by_key( &self, object_id: &ObjectID, @@ -608,24 +565,10 @@ impl AuthorityStore { // Update the index if object.get_single_owner().is_some() { - match object.owner { - Owner::AddressOwner(addr) => self - .perpetual_tables - .owner_index - .insert(&(addr, object_ref.0), &ObjectInfo::new(&object_ref, object))?, - Owner::ObjectOwner(object_id) => { - if let Some(info) = self.try_create_dynamic_field_info( - &object_ref, - object, - &Default::default(), - )? { - self.perpetual_tables - .dynamic_field_index - .insert(&(ObjectID::from(object_id), object_ref.0), &info)? - } - } - _ => {} - } + self.perpetual_tables.owner_index.insert( + &(object.owner, object_ref.0), + &ObjectInfo::new(&object_ref, object), + )?; // Only initialize lock for address owned objects. if !object.is_child_object() { self.lock_service @@ -633,6 +576,7 @@ impl AuthorityStore { .await?; } } + // Update the parent self.perpetual_tables .parent_sync @@ -641,66 +585,6 @@ impl AuthorityStore { Ok(()) } - fn try_create_dynamic_field_info( - &self, - oref: &ObjectRef, - o: &Object, - uncommitted_objects: &BTreeMap, - ) -> SuiResult> { - // Skip if not a move object - let Some(move_object) = o.data.try_as_move() else { - return Ok(None); - }; - // We only index dynamic field objects - if !DynamicFieldInfo::is_dynamic_field(&move_object.type_) { - return Ok(None); - } - let move_struct = - move_object.to_move_struct_with_resolver(ObjectFormatOptions::default(), &self)?; - - let Some((name, type_, object_id)) = DynamicFieldInfo::parse_move_object(&move_struct)? else{ - return Ok(None) - }; - - Ok(Some(match type_ { - DynamicFieldType::DynamicObject => { - // Find the actual object from storage using the object id obtained from the wrapper. - let (object_type, version, digest) = - if let Some(o) = uncommitted_objects.get(&object_id) { - o.data - .type_() - .map(|type_| (type_.clone(), o.version(), o.digest())) - } else if let Ok(Some(o)) = self.get_object(&object_id) { - o.data - .type_() - .map(|type_| (type_.clone(), o.version(), o.digest())) - } else { - None - } - .ok_or_else(|| SuiError::ObjectDeserializationError { - error: format!("Cannot found data for dynamic object {object_id}"), - })?; - - DynamicFieldInfo { - name, - type_, - object_type: object_type.to_string(), - object_id, - version, - digest, - } - } - DynamicFieldType::DynamicField { .. } => DynamicFieldInfo { - name, - type_, - object_type: move_object.type_.type_params[1].to_string(), - object_id: oref.0, - version: oref.1, - digest: oref.2, - }, - })) - } - /// This function is used by the bench.rs script, and should not be used in other contexts /// In particular it does not check the old locks before inserting new ones, so the objects /// must be new. @@ -711,38 +595,6 @@ impl AuthorityStore { .map(|o| (o.compute_object_reference(), o)) .collect(); - let (owner_insert, dynamic_field_insert): (Vec<_>, Vec<_>) = ref_and_objects - .iter() - .map(|(object_ref, o)| match o.owner { - Owner::AddressOwner(address) => ( - Some(((address, o.id()), ObjectInfo::new(object_ref, o))), - None, - ), - Owner::ObjectOwner(object_id) => ( - None, - Some(((ObjectID::from(object_id), o.id()), (object_ref, o))), - ), - _ => (None, None), - }) - .unzip(); - - let owner_insert = owner_insert.into_iter().flatten(); - - let uncommitted_objects = ref_and_objects - .iter() - .map(|((id, ..), o)| (*id, **o)) - .collect(); - - let dynamic_field_insert = dynamic_field_insert - .into_iter() - .flatten() - .flat_map(|(key, (oref, o))| { - self.try_create_dynamic_field_info(oref, o, &uncommitted_objects) - .transpose() - .map(|info| info.map(|info| (key, info))) - }) - .collect::>>()?; - batch .insert_batch( &self.perpetual_tables.objects, @@ -750,10 +602,11 @@ impl AuthorityStore { .iter() .map(|(oref, o)| (ObjectKey::from(oref), **o)), )? - .insert_batch(&self.perpetual_tables.owner_index, owner_insert)? .insert_batch( - &self.perpetual_tables.dynamic_field_index, - dynamic_field_insert, + &self.perpetual_tables.owner_index, + ref_and_objects + .iter() + .map(|(oref, o)| ((o.owner, oref.0), ObjectInfo::new(oref, o))), )? .insert_batch( &self.perpetual_tables.parent_sync, @@ -969,7 +822,7 @@ impl AuthorityStore { // For wrapped objects, although their owners technically didn't change, we will lose track // of them and there is no guarantee on their owner in the future. Hence we treat them // the same as deleted. - let (old_object_owners, old_dynamic_fields): (Vec<_>, Vec<_>) = + let old_object_owners = deleted .iter() // We need to call get() on objects because some object that were just deleted may not @@ -983,24 +836,11 @@ impl AuthorityStore { } _ => None, }, - )) - .map(|(owner, id)| match owner { - Owner::AddressOwner(address) => (Some((address, id)), None), - Owner::ObjectOwner(object_id) => (None, Some((ObjectID::from(object_id), id))), - _ => (None, None), - }) - .unzip(); - - let old_object_owners = old_object_owners.into_iter().flatten(); - let old_dynamic_fields = old_dynamic_fields.into_iter().flatten(); + )); // Delete the old owner index entries write_batch = write_batch.delete_batch(&self.perpetual_tables.owner_index, old_object_owners)?; - write_batch = write_batch.delete_batch( - &self.perpetual_tables.dynamic_field_index, - old_dynamic_fields, - )?; // Index the certificate by the objects mutated write_batch = write_batch.insert_batch( @@ -1028,40 +868,20 @@ impl AuthorityStore { ) }), )?; - let uncommitted_objects = written.iter().map(|(id, (_, o, _))| (*id, o)).collect(); - // Update the indexes of the objects written - let (owner_written, dynamic_field_written): (Vec<_>, Vec<_>) = written - .iter() - .map(|(id, (object_ref, new_object, _))| match new_object.owner { - Owner::AddressOwner(address) => ( - Some(((address, *id), ObjectInfo::new(object_ref, new_object))), - None, - ), - Owner::ObjectOwner(object_id) => ( - None, - Some(((ObjectID::from(object_id), *id), (object_ref, new_object))), - ), - _ => (None, None), - }) - .unzip(); - - let owner_written = owner_written.into_iter().flatten(); - let dynamic_field_written = dynamic_field_written - .into_iter() - .flatten() - .flat_map(|(key, (oref, o))| { - self.try_create_dynamic_field_info(oref, o, &uncommitted_objects) - .transpose() - .map(|info| info.map(|info| (key, info))) - }) - .collect::>>()?; - write_batch = - write_batch.insert_batch(&self.perpetual_tables.owner_index, owner_written)?; + // Update the indexes of the objects written write_batch = write_batch.insert_batch( - &self.perpetual_tables.dynamic_field_index, - dynamic_field_written, + &self.perpetual_tables.owner_index, + written + .iter() + .filter_map(|(_id, (object_ref, new_object, _kind))| { + trace!(?object_ref, owner =? new_object.owner, "Updating owner_index"); + new_object + .get_owner_and_id() + .map(|owner_id| (owner_id, ObjectInfo::new(object_ref, new_object))) + }), )?; + // Insert each output object into the stores write_batch = write_batch.insert_batch( &self.perpetual_tables.objects, @@ -1177,83 +997,62 @@ impl AuthorityStore { // the older version, and then rewrite the entry with the old object info. // TODO: Validators should not need to maintain owner_index. // This is dependent on https://github.com/MystenLabs/sui/issues/2629. - let (owners_to_delete, dynamic_field_to_delete): (Vec<_>, Vec<_>) = effects + let owners_to_delete = effects .created .iter() .chain(effects.unwrapped.iter()) .chain(effects.mutated.iter()) - .map(|((id, _, _), owner)| match owner { - Owner::AddressOwner(addr) => (Some((*addr, *id)), None), - Owner::ObjectOwner(object_id) => (None, Some((ObjectID::from(*object_id), *id))), - _ => (None, None), - }) - .unzip(); - - let owners_to_delete = owners_to_delete.into_iter().flatten(); - let dynamic_field_to_delete = dynamic_field_to_delete.into_iter().flatten(); - + .map(|((id, _, _), owner)| (*owner, *id)); write_batch = write_batch.delete_batch(&self.perpetual_tables.owner_index, owners_to_delete)?; - write_batch = write_batch.delete_batch( - &self.perpetual_tables.dynamic_field_index, - dynamic_field_to_delete, - )?; + let modified_object_keys = effects .modified_at_versions .iter() .map(|(id, version)| ObjectKey(*id, *version)); - let (old_objects_and_locks, old_dynamic_fields): (Vec<_>, Vec<_>) = self + let (old_modified_objects, old_locks): (Vec<_>, Vec<_>) = self .perpetual_tables .objects .multi_get(modified_object_keys)? .into_iter() - .map(|obj_opt| { + .filter_map(|obj_opt| { let obj = obj_opt.expect("Older object version not found"); - match obj.owner { - Owner::AddressOwner(addr) => { - let oref = obj.compute_object_reference(); - ( - Some((((addr, obj.id()), ObjectInfo::new(&oref, &obj)), oref)), - None, - ) - } - Owner::ObjectOwner(object_id) => { - (None, Some(((ObjectID::from(object_id), obj.id()), obj))) - } - _ => (None, None), + + if obj.is_immutable() { + return None; } + + let obj_ref = obj.compute_object_reference(); + Some(( + ((obj.owner, obj.id()), ObjectInfo::new(&obj_ref, &obj)), + obj.is_address_owned().then_some(obj_ref), + )) }) .unzip(); - let (old_modified_objects, old_locks): (Vec<_>, Vec<_>) = - old_objects_and_locks.into_iter().flatten().unzip(); - let old_dynamic_fields = old_dynamic_fields - .into_iter() - .flatten() - .flat_map(|(key, o)| { - self.try_create_dynamic_field_info( - &o.compute_object_reference(), - &o, - &Default::default(), - ) - .transpose() - .map(|info| info.map(|info| (key, info))) - }) - .collect::>>()?; + let old_locks: Vec<_> = old_locks.into_iter().flatten().collect(); write_batch = write_batch.insert_batch(&self.perpetual_tables.owner_index, old_modified_objects)?; - write_batch = write_batch.insert_batch( - &self.perpetual_tables.dynamic_field_index, - old_dynamic_fields, - )?; write_batch.write()?; self.lock_service.initialize_locks(&old_locks, true).await?; Ok(()) } + /// Return the object with version less then or eq to the provided seq number. + /// This is used by indexer to find the correct version of dynamic field child object. + /// We do not store the version of the child object, but because of lamport timestamp, + /// we know the child must have version number less then or eq to the parent. + pub fn find_object_lt_or_eq_version( + &self, + object_id: ObjectID, + version: SequenceNumber, + ) -> Option { + self.perpetual_tables + .find_object_lt_or_eq_version(object_id, version) + } /// Returns the last entry we have for this object in the parents_sync index used /// to facilitate client and authority sync. In turn the latest entry provides the @@ -1638,30 +1437,6 @@ impl ModuleResolver for AuthorityStore { } } -impl Deserialize<'de>> GetModule for &SuiDataStore { - type Error = SuiError; - type Item = Arc; - - fn get_module_by_id(&self, id: &ModuleId) -> Result>, Self::Error> { - if let Some(compiled_module) = self.module_cache.read().get(id) { - return Ok(Some(compiled_module.clone())); - } - - if let Some(module_bytes) = self.get_module(id)? { - let module = Arc::new(CompiledModule::deserialize(&module_bytes).map_err(|e| { - SuiError::ModuleDeserializationFailure { - error: e.to_string(), - } - })?); - - self.module_cache.write().insert(id.clone(), module.clone()); - Ok(Some(module)) - } else { - Ok(None) - } - } -} - /// A wrapper to make Orphan Rule happy pub struct ResolverWrapper(pub Arc); diff --git a/crates/sui-core/src/authority/authority_store_tables.rs b/crates/sui-core/src/authority/authority_store_tables.rs index f440bec9413f1..178ef72469391 100644 --- a/crates/sui-core/src/authority/authority_store_tables.rs +++ b/crates/sui-core/src/authority/authority_store_tables.rs @@ -7,7 +7,6 @@ use std::path::Path; use sui_storage::default_db_options; use sui_types::base_types::{ExecutionDigests, SequenceNumber}; use sui_types::batch::{SignedBatch, TxSequenceNumber}; -use sui_types::dynamic_field::DynamicFieldInfo; use sui_types::messages::TrustedCertificate; use typed_store::rocks::{DBMap, DBOptions}; use typed_store::traits::TypedStoreDebug; @@ -31,17 +30,11 @@ pub struct AuthorityPerpetualTables { #[default_options_override_fn = "objects_table_default_config"] pub(crate) objects: DBMap, - /// This is an index of object references to currently existing objects, indexed by the + /// This is a an index of object references to currently existing objects, indexed by the /// composite key of the SuiAddress of their owner and the object ID of the object. /// This composite index allows an efficient iterator to list all objected currently owned /// by a specific user, and their object reference. - pub(crate) owner_index: DBMap<(SuiAddress, ObjectID), ObjectInfo>, - - /// This is an index of object references to currently existing dynamic field object, indexed by the - /// composite key of the object ID of their parent and the object ID of the dynamic field object. - /// This composite index allows an efficient iterator to list all objects currently owned - /// by a specific object, and their object reference. - pub(crate) dynamic_field_index: DBMap<(ObjectID, ObjectID), DynamicFieldInfo>, + pub(crate) owner_index: DBMap<(Owner, ObjectID), ObjectInfo>, /// This is a map between the transaction digest and the corresponding certificate for all /// certificates that have been successfully processed by this authority. This set of certificates @@ -139,6 +132,22 @@ impl AuthorityPerpetualTables { } } + // This is used by indexer to find the correct version of dynamic field child object. + // We do not store the version of the child object, but because of lamport timestamp, + // we know the child must have version number less then or eq to the parent. + pub fn find_object_lt_or_eq_version( + &self, + object_id: ObjectID, + version: SequenceNumber, + ) -> Option { + let Ok(iter) = self.objects + .iter() + .skip_prior_to(&ObjectKey(object_id, version))else { + return None + }; + iter.reverse().next().map(|(_, o)| o) + } + pub fn get_latest_parent_entry( &self, object_id: ObjectID, diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index 5c6d249132bff..f3137f1443c0e 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -43,7 +43,6 @@ use sui_types::{ }; use sui_types::{crypto::AuthorityPublicKeyBytes, object::Data}; -use narwhal_types::Certificate; use sui_types::dynamic_field::DynamicFieldType; use tracing::info; @@ -2457,8 +2456,16 @@ async fn test_store_revert_transfer_sui() { db.get_latest_parent_entry(gas_object_id).unwrap().unwrap(), (gas_object_ref, TransactionDigest::genesis()), ); - assert!(db.get_owner_objects(recipient).unwrap().is_empty()); - assert_eq!(db.get_owner_objects(sender).unwrap().len(), 1); + assert!(db + .get_owner_objects(Owner::AddressOwner(recipient)) + .unwrap() + .is_empty()); + assert_eq!( + db.get_owner_objects(Owner::AddressOwner(sender)) + .unwrap() + .len(), + 1 + ); assert!(db.get_certified_transaction(&tx_digest).unwrap().is_none()); assert!(db.as_ref().get_effects(&tx_digest).is_err()); } diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index ef6cb983530a3..d8ee7a6c7b7b9 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -75,17 +75,19 @@ impl RpcReadApiServer for ReadApi { .collect()) } + // TODO: Remove this + // This is very expensive, it's only for backward compatibilities and should be removed asap. async fn get_objects_owned_by_object( &self, object_id: ObjectID, ) -> RpcResult> { - let dynamic_fields: DynamicFieldPage = - self.get_dynamic_fields(object_id, None, None).await?; + let dynamic_fields = self + .state + .get_dynamic_fields(object_id, None, usize::MAX) + .map_err(|e| anyhow!("{e}"))?; let mut object_info = vec![]; - for info in dynamic_fields.data { - // TODO: Remove this - // This is very expensive, it's only for backward compatibilities and should be removed asap. + for info in dynamic_fields { let object = self .state .get_object_read(&info.object_id) diff --git a/crates/sui-storage/src/indexes.rs b/crates/sui-storage/src/indexes.rs index 74f00c3b4ebbe..c98684f7940fd 100644 --- a/crates/sui-storage/src/indexes.rs +++ b/crates/sui-storage/src/indexes.rs @@ -6,20 +6,32 @@ use move_core_types::identifier::Identifier; use serde::{de::DeserializeOwned, Serialize}; -use typed_store::rocks::DBMap; -use typed_store::rocks::DBOptions; -use typed_store::traits::Map; -use typed_store::traits::TypedStoreDebug; -use typed_store_derive::DBMapUtils; +use tracing::debug; -use sui_types::base_types::ObjectRef; use sui_types::base_types::{ObjectID, SuiAddress, TransactionDigest}; +use sui_types::base_types::{ObjectInfo, ObjectRef}; use sui_types::batch::TxSequenceNumber; +use sui_types::dynamic_field::DynamicFieldInfo; use sui_types::error::SuiResult; use sui_types::object::Owner; +use typed_store::rocks::DBMap; +use typed_store::rocks::DBOptions; +use typed_store::traits::Map; +use typed_store::traits::TypedStoreDebug; +use typed_store_derive::DBMapUtils; use crate::default_db_options; +type OwnerIndexKey = (SuiAddress, ObjectID); +type DynamicFieldKey = (ObjectID, ObjectID); + +pub struct ObjectIndexChanges { + pub deleted_owners: Vec, + pub deleted_dynamic_fields: Vec, + pub new_owners: Vec<(OwnerIndexKey, ObjectInfo)>, + pub new_dynamic_fields: Vec<(DynamicFieldKey, DynamicFieldInfo)>, +} + #[derive(DBMapUtils)] pub struct IndexStore { /// Index from sui address to transactions initiated by that address. @@ -53,6 +65,20 @@ pub struct IndexStore { /// Index from transaction digest to sequence number. #[default_options_override_fn = "transactions_seq_table_default_config"] transactions_seq: DBMap, + + /// This is an index of object references to currently existing objects, indexed by the + /// composite key of the SuiAddress of their owner and the object ID of the object. + /// This composite index allows an efficient iterator to list all objected currently owned + /// by a specific user, and their object reference. + #[default_options_override_fn = "owner_index_table_default_config"] + owner_index: DBMap, + + /// This is an index of object references to currently existing dynamic field object, indexed by the + /// composite key of the object ID of their parent and the object ID of the dynamic field object. + /// This composite index allows an efficient iterator to list all objects currently owned + /// by a specific object, and their object reference. + #[default_options_override_fn = "dynamic_field_index_table_default_config"] + dynamic_field_index: DBMap, } // These functions are used to initialize the DB tables @@ -77,6 +103,12 @@ fn transactions_by_move_function_table_default_config() -> DBOptions { fn timestamps_table_default_config() -> DBOptions { default_db_options(None, Some(1_000_000)).1 } +fn owner_index_table_default_config() -> DBOptions { + default_db_options(None, Some(1_000_000)).0 +} +fn dynamic_field_index_table_default_config() -> DBOptions { + default_db_options(None, Some(1_000_000)).0 +} impl IndexStore { pub fn index_tx( @@ -85,6 +117,7 @@ impl IndexStore { active_inputs: impl Iterator, mutated_objects: impl Iterator + Clone, move_functions: impl Iterator + Clone, + object_index_changes: ObjectIndexChanges, sequence: TxSequenceNumber, digest: &TransactionDigest, timestamp_ms: u64, @@ -134,6 +167,24 @@ impl IndexStore { let batch = batch.insert_batch(&self.timestamps, std::iter::once((*digest, timestamp_ms)))?; + // Owner index + let batch = batch.delete_batch( + &self.owner_index, + object_index_changes.deleted_owners.into_iter(), + )?; + let batch = batch.delete_batch( + &self.dynamic_field_index, + object_index_changes.deleted_dynamic_fields.into_iter(), + )?; + let batch = batch.insert_batch( + &self.owner_index, + object_index_changes.new_owners.into_iter(), + )?; + let batch = batch.insert_batch( + &self.dynamic_field_index, + object_index_changes.new_dynamic_fields.into_iter(), + )?; + batch.write()?; Ok(()) @@ -293,4 +344,50 @@ impl IndexStore { ) -> SuiResult> { Ok(self.transactions_seq.get(digest)?) } + + pub fn get_dynamic_fields( + &self, + object: ObjectID, + cursor: Option, + limit: usize, + ) -> SuiResult> { + debug!(?object, "get_dynamic_fields"); + let cursor = cursor.unwrap_or(ObjectID::ZERO); + Ok(self + .dynamic_field_index + .iter() + // The object id 0 is the smallest possible + .skip_to(&(object, cursor))? + .take_while(|((object_owner, _), _)| (object_owner == &object)) + .map(|(_, object_info)| object_info) + .take(limit) + .collect()) + } + + pub fn get_dynamic_field_object_id( + &self, + object: ObjectID, + name: &str, + ) -> SuiResult> { + debug!(?object, "get_dynamic_field_object_id"); + Ok(self + .dynamic_field_index + .iter() + // The object id 0 is the smallest possible + .skip_to(&(object, ObjectID::ZERO))? + .find(|((object_owner, _), info)| (object_owner == &object && info.name == name)) + .map(|(_, object_info)| object_info.object_id)) + } + + pub fn get_owner_objects(&self, owner: SuiAddress) -> SuiResult> { + debug!(?owner, "get_owner_objects"); + Ok(self + .owner_index + .iter() + // The object id 0 is the smallest possible + .skip_to(&(owner, ObjectID::ZERO))? + .take_while(|((object_owner, _), _)| (object_owner == &owner)) + .map(|(_, object_info)| object_info) + .collect()) + } } diff --git a/crates/sui-types/src/error.rs b/crates/sui-types/src/error.rs index 28fcd16cbdbf6..b7f3e06dfcef6 100644 --- a/crates/sui-types/src/error.rs +++ b/crates/sui-types/src/error.rs @@ -524,6 +524,9 @@ pub enum SuiError { #[error("SUI payment transactions use first input coin for gas payment, but found a different gas object.")] UnexpectedGasPaymentObject, + #[error("Index store not available on this Fullnode.")] + IndexStoreNotAvailable, + #[error("unknown error: {0}")] Unknown(String), } From 24b806d740b21f50693554119c8938c020e228eb Mon Sep 17 00:00:00 2001 From: patrick Date: Wed, 7 Dec 2022 22:41:23 +0000 Subject: [PATCH 11/13] create genesis owner index --- crates/sui-core/src/authority.rs | 144 ++++++++++++------ .../sui-core/src/authority/authority_store.rs | 3 +- crates/sui-json-rpc/src/api.rs | 1 - crates/sui-sdk/src/lib.rs | 2 - crates/sui-storage/src/indexes.rs | 18 +++ crates/sui-types/src/dynamic_field.rs | 18 +-- 6 files changed, 117 insertions(+), 69 deletions(-) diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index 32d662eca3967..544d701149971 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -2,6 +2,17 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::cmp::Ordering as CmpOrdering; +use std::hash::Hash; +use std::path::PathBuf; +use std::str::FromStr; +use std::time::Duration; +use std::{ + collections::{HashMap, VecDeque}, + pin::Pin, + sync::{atomic::Ordering, Arc}, +}; + use anyhow::anyhow; use arc_swap::Guard; use chrono::prelude::*; @@ -17,16 +28,6 @@ use prometheus::{ exponential_buckets, register_histogram_with_registry, register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, IntCounter, IntGauge, Registry, }; -use std::cmp::Ordering as CmpOrdering; -use std::hash::Hash; -use std::path::PathBuf; -use std::str::FromStr; -use std::time::Duration; -use std::{ - collections::{HashMap, VecDeque}, - pin::Pin, - sync::{atomic::Ordering, Arc}, -}; use tap::TapFallible; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc::unbounded_channel; @@ -1257,7 +1258,9 @@ impl AuthorityState { effects: &SignedTransactionEffects, timestamp_ms: u64, ) -> SuiResult { - let changes = self.process_object_index(effects)?; + let changes = self + .process_object_index(effects) + .tap_err(|e| warn!("{e}"))?; indexes.index_tx( cert.sender_address(), @@ -1288,10 +1291,20 @@ impl AuthorityState { &self, effects: &SignedTransactionEffects, ) -> Result { + let modified_at_version = effects + .modified_at_versions + .iter() + .cloned() + .collect::>(); + let mut deleted_owners = vec![]; let mut deleted_dynamic_fields = vec![]; - for (id, version, _) in &effects.deleted { - match self.get_owner_at_version(id, *version)? { + for (id, _, _) in &effects.deleted { + let Some(old_version) = modified_at_version.get(id) else{ + error!("Error processing object owner index for tx [{}], cannot find modified at version for deleted object [{id}].", effects.transaction_digest); + continue; + }; + match self.get_owner_at_version(id, *old_version)? { Owner::AddressOwner(addr) => deleted_owners.push((addr, *id)), Owner::ObjectOwner(object_id) => { deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) @@ -1303,12 +1316,6 @@ impl AuthorityState { let mut new_owners = vec![]; let mut new_dynamic_fields = vec![]; - let modified_at_version = effects - .modified_at_versions - .iter() - .cloned() - .collect::>(); - for (oref, owner, kind) in effects.all_mutated() { let id = &oref.0; // For mutated objects, retrieve old owner and delete old index if there is a owner change. @@ -1321,19 +1328,14 @@ impl AuthorityState { error!("Error processing object owner index for tx [{}], cannot find object [{id}] at version [{old_version}].", effects.transaction_digest); continue; }; - if &old_object.owner != owner { - match old_object.owner { - Owner::AddressOwner(addr) => { - deleted_owners.push((addr, *id)); - } - Owner::ObjectOwner(object_id) => { - deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) - } - _ => {} + match old_object.owner { + Owner::AddressOwner(addr) => { + deleted_owners.push((addr, *id)); } - } else { - // Owner is the same, nothing to update. - continue; + Owner::ObjectOwner(object_id) => { + deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) + } + _ => {} } } @@ -1361,7 +1363,10 @@ impl AuthorityState { )); } Owner::ObjectOwner(owner) => { - let Some(df_info) = self.try_create_dynamic_field_info(oref)? else{ + let Some(o) = self.database.get_object_by_key(&oref.0, oref.1)? else{ + continue; + }; + let Some(df_info) = self.try_create_dynamic_field_info(o)? else{ continue; }; new_dynamic_fields.push(((ObjectID::from(*owner), *id), df_info)) @@ -1378,13 +1383,9 @@ impl AuthorityState { }) } - fn try_create_dynamic_field_info( - &self, - oref: &ObjectRef, - ) -> SuiResult> { + fn try_create_dynamic_field_info(&self, o: Object) -> SuiResult> { // Skip if not a move object - let Some(move_object) = self - .database.get_object_by_key(&oref.0, oref.1)?.and_then(|o| o.data.try_as_move().cloned()) else { + let Some(move_object) = o.data.try_as_move().cloned() else { return Ok(None); }; // We only index dynamic field objects @@ -1396,17 +1397,16 @@ impl AuthorityState { self.module_cache.as_ref(), )?; - let Some((name, type_, object_id)) = DynamicFieldInfo::parse_move_object(&move_struct)? else{ - return Ok(None) - }; + let (name, type_, object_id) = + DynamicFieldInfo::parse_move_object(&move_struct).tap_err(|e| warn!("{e}"))?; Ok(Some(match type_ { DynamicFieldType::DynamicObject => { // Find the actual object from storage using the object id obtained from the wrapper. - let Some(object) = self.database.find_object_lt_or_eq_version(object_id, oref.1) else{ + let Some(object) = self.database.find_object_lt_or_eq_version(object_id, o.version()) else{ return Err(SuiError::ObjectNotFound { object_id, - version: Some(oref.1), + version: Some(o.version()), }) }; let version = object.version(); @@ -1426,9 +1426,9 @@ impl AuthorityState { name, type_, object_type: move_object.type_.type_params[1].to_string(), - object_id: oref.0, - version: oref.1, - digest: oref.2, + object_id: o.id(), + version: o.version(), + digest: o.digest(), }, })) } @@ -1768,6 +1768,10 @@ impl AuthorityState { let authority_state = Arc::downgrade(&state); spawn_monitored_task!(execution_process(authority_state, rx_ready_certificates)); + state + .create_owner_index_if_empty() + .expect("Error indexing genesis objects."); + state } @@ -1818,19 +1822,29 @@ impl AuthorityState { None, )); + let index_store = Some(Arc::new(IndexStore::open_tables_read_write( + path.join("indexes"), + None, + None, + ))); + // add the object_basics module - AuthorityState::new( + let state = AuthorityState::new( secret.public().into(), secret.clone(), store, node_sync_store, epochs, - None, + index_store, None, None, &Registry::new(), ) - .await + .await; + + state.create_owner_index_if_empty().unwrap(); + + state } /// Adds certificates to the pending certificate store and transaction manager for ordered execution. @@ -1879,6 +1893,38 @@ impl AuthorityState { Ok(()) } + fn create_owner_index_if_empty(&self) -> SuiResult { + let Some(index_store) = &self.indexes else{ + return Ok(()) + }; + + let mut new_owners = vec![]; + let mut new_dynamic_fields = vec![]; + for (_, o) in self.database.perpetual_tables.objects.iter() { + match o.owner { + Owner::AddressOwner(addr) => new_owners.push(( + (addr, o.id()), + ObjectInfo::new(&o.compute_object_reference(), &o), + )), + Owner::ObjectOwner(object_id) => { + let id = o.id(); + let Some(info) = self.try_create_dynamic_field_info(o)? else{ + continue; + }; + new_dynamic_fields.push(((ObjectID::from(object_id), id), info)); + } + _ => {} + } + } + + index_store.insert_genesis_objects(ObjectIndexChanges { + deleted_owners: vec![], + deleted_dynamic_fields: vec![], + new_owners, + new_dynamic_fields, + }) + } + pub fn reconfigure(&self, new_committee: Committee) -> SuiResult { // TODO: We should move the committee into epoch db store, so that the operation below // can become atomic. diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index 28143c1d53613..e526cd8bf23e0 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -1,7 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::iter; use std::path::Path; use std::sync::Arc; @@ -15,7 +15,6 @@ use serde_with::serde_as; use tokio_retry::strategy::{jitter, ExponentialBackoff}; use tracing::{debug, info, trace}; -use narwhal_executor::ExecutionIndices; use sui_storage::{ lock_service::ObjectLockStatus, mutex_table::{LockGuard, MutexTable}, diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index 1f22afcef8c7d..96e1b58fb0ec2 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -3,7 +3,6 @@ use std::collections::BTreeMap; -use anyhow::anyhow; use fastcrypto::encoding::Base64; use jsonrpsee::core::RpcResult; use jsonrpsee_proc_macros::rpc; diff --git a/crates/sui-sdk/src/lib.rs b/crates/sui-sdk/src/lib.rs index 91276e2bba34b..bd3aa06aced3d 100644 --- a/crates/sui-sdk/src/lib.rs +++ b/crates/sui-sdk/src/lib.rs @@ -15,8 +15,6 @@ use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; use crate::error::{RpcError, SuiRpcResult}; use rpc_types::{SuiCertifiedTransaction, SuiParsedTransactionResponse, SuiTransactionEffects}; use serde_json::Value; -pub use sui_config::gateway; -use sui_core::gateway_state::TxSeqNumber; pub use sui_json as json; use crate::apis::{CoinReadApi, EventApi, QuorumDriver, ReadApi}; diff --git a/crates/sui-storage/src/indexes.rs b/crates/sui-storage/src/indexes.rs index c98684f7940fd..edc05d4c86910 100644 --- a/crates/sui-storage/src/indexes.rs +++ b/crates/sui-storage/src/indexes.rs @@ -390,4 +390,22 @@ impl IndexStore { .map(|(_, object_info)| object_info) .collect()) } + + pub fn insert_genesis_objects(&self, object_index_changes: ObjectIndexChanges) -> SuiResult { + let batch = self.owner_index.batch(); + let batch = batch.insert_batch( + &self.owner_index, + object_index_changes.new_owners.into_iter(), + )?; + let batch = batch.insert_batch( + &self.dynamic_field_index, + object_index_changes.new_dynamic_fields.into_iter(), + )?; + batch.write()?; + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.owner_index.is_empty() + } } diff --git a/crates/sui-types/src/dynamic_field.rs b/crates/sui-types/src/dynamic_field.rs index 40995f0501b66..ac0cb26708004 100644 --- a/crates/sui-types/src/dynamic_field.rs +++ b/crates/sui-types/src/dynamic_field.rs @@ -41,7 +41,7 @@ impl DynamicFieldInfo { pub fn parse_move_object( move_struct: &MoveStruct, - ) -> SuiResult> { + ) -> SuiResult<(String, DynamicFieldType, ObjectID)> { let name = extract_field_from_move_struct(move_struct, "name").ok_or_else(|| { SuiError::ObjectDeserializationError { error: "Cannot extract [name] field from sui::dynamic_field::Field".to_string(), @@ -54,18 +54,6 @@ impl DynamicFieldInfo { } })?; - // Extract value from value option - let Some(value) = (match &value { - MoveValue::Struct(MoveStruct::WithTypes { type_: _, fields }) => match fields.first() { - Some((_, MoveValue::Vector(v))) => v.first().cloned(), - _ => None, - }, - _ => None, - }) else { - // Skip processing if Field's value is not set. - return Ok(None) - }; - let object_id = extract_object_id(&value).ok_or_else(|| SuiError::ObjectDeserializationError { error: format!( @@ -74,7 +62,7 @@ impl DynamicFieldInfo { ), })?; - Ok(Some(if is_dynamic_object(move_struct) { + Ok(if is_dynamic_object(move_struct) { let name = match name { MoveValue::Struct(s) => extract_field_from_move_struct(&s, "name"), _ => None, @@ -93,7 +81,7 @@ impl DynamicFieldInfo { }, object_id, ) - })) + }) } } From bda0c6af9c60858a6b2f1ee45c9eb93254eb8066 Mon Sep 17 00:00:00 2001 From: patrick Date: Fri, 9 Dec 2022 10:11:11 +0000 Subject: [PATCH 12/13] clean up, revert import reordering. --- crates/sui-core/src/authority.rs | 53 +++--- .../sui-core/src/authority/authority_store.rs | 30 ++-- .../src/unit_tests/authority_tests.rs | 4 +- .../src/unit_tests/move_integration_tests.rs | 1 - crates/sui-json-rpc-types/src/lib.rs | 2 +- crates/sui-json-rpc/src/api.rs | 23 ++- crates/sui-json-rpc/src/coin_api.rs | 6 +- crates/sui-json-rpc/src/read_api.rs | 6 +- crates/sui-open-rpc/spec/openrpc.json | 161 ++++++++++++++++++ crates/sui-storage/src/indexes.rs | 23 ++- 10 files changed, 239 insertions(+), 70 deletions(-) diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index 544d701149971..16f78fb9b4f3b 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -2,17 +2,6 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::cmp::Ordering as CmpOrdering; -use std::hash::Hash; -use std::path::PathBuf; -use std::str::FromStr; -use std::time::Duration; -use std::{ - collections::{HashMap, VecDeque}, - pin::Pin, - sync::{atomic::Ordering, Arc}, -}; - use anyhow::anyhow; use arc_swap::Guard; use chrono::prelude::*; @@ -28,14 +17,24 @@ use prometheus::{ exponential_buckets, register_histogram_with_registry, register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, IntCounter, IntGauge, Registry, }; +use std::cmp::Ordering as CmpOrdering; +use std::hash::Hash; +use std::path::PathBuf; +use std::str::FromStr; +use std::time::Duration; +use std::{ + collections::{HashMap, VecDeque}, + pin::Pin, + sync::{atomic::Ordering, Arc}, +}; use tap::TapFallible; use tokio::sync::broadcast::error::RecvError; use tokio::sync::mpsc::unbounded_channel; use tracing::{debug, error, instrument, warn, Instrument}; +use typed_store::Map; pub use authority_notify_read::EffectsNotifyRead; pub use authority_store::{AuthorityStore, ResolverWrapper, UpdateType}; -use mysten_metrics::monitored_scope; use narwhal_config::{ Committee as ConsensusCommittee, WorkerCache as ConsensusWorkerCache, WorkerId as ConsensusWorkerId, @@ -80,7 +79,6 @@ use sui_types::{ storage::{BackingPackageStore, DeleteKind}, MOVE_STDLIB_ADDRESS, SUI_FRAMEWORK_ADDRESS, SUI_SYSTEM_STATE_OBJECT_ID, }; -use typed_store::Map; use crate::authority::authority_notifier::TransactionNotifierTicket; use crate::authority::authority_notify_read::NotifyRead; @@ -1328,19 +1326,22 @@ impl AuthorityState { error!("Error processing object owner index for tx [{}], cannot find object [{id}] at version [{old_version}].", effects.transaction_digest); continue; }; - match old_object.owner { - Owner::AddressOwner(addr) => { - deleted_owners.push((addr, *id)); - } - Owner::ObjectOwner(object_id) => { - deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) + if &old_object.owner != owner { + match old_object.owner { + Owner::AddressOwner(addr) => { + deleted_owners.push((addr, *id)); + } + Owner::ObjectOwner(object_id) => { + deleted_dynamic_fields.push((ObjectID::from(object_id), *id)) + } + _ => {} } - _ => {} } } match owner { Owner::AddressOwner(addr) => { + // TODO: We can remove the object fetching after we added ObjectType to TransactionEffects let Some(o) = self.database.get_object_by_key(id, oref.1)? else{ continue; }; @@ -1367,6 +1368,7 @@ impl AuthorityState { continue; }; let Some(df_info) = self.try_create_dynamic_field_info(o)? else{ + // Skip indexing for non dynamic field objects. continue; }; new_dynamic_fields.push(((ObjectID::from(*owner), *id), df_info)) @@ -2084,6 +2086,17 @@ impl AuthorityState { } } + pub fn get_owner_objects_iterator( + &self, + owner: SuiAddress, + ) -> SuiResult + '_> { + if let Some(indexes) = &self.indexes { + indexes.get_owner_objects_iterator(owner) + } else { + Err(SuiError::IndexStoreNotAvailable) + } + } + pub fn get_dynamic_fields( &self, owner: ObjectID, diff --git a/crates/sui-core/src/authority/authority_store.rs b/crates/sui-core/src/authority/authority_store.rs index e526cd8bf23e0..502b9d619bc09 100644 --- a/crates/sui-core/src/authority/authority_store.rs +++ b/crates/sui-core/src/authority/authority_store.rs @@ -1,42 +1,34 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeSet; -use std::iter; -use std::path::Path; -use std::sync::Arc; -use std::{fmt::Debug, path::PathBuf}; - +use super::{authority_store_tables::AuthorityPerpetualTables, *}; +use crate::authority::authority_per_epoch_store::{ + AuthorityPerEpochStore, ExecutionIndicesWithHash, +}; use arc_swap::ArcSwap; use once_cell::sync::OnceCell; use rocksdb::Options; use serde::{Deserialize, Serialize}; use serde_with::serde_as; -use tokio_retry::strategy::{jitter, ExponentialBackoff}; -use tracing::{debug, info, trace}; - +use std::collections::BTreeSet; +use std::iter; +use std::path::Path; +use std::sync::Arc; +use std::{fmt::Debug, path::PathBuf}; use sui_storage::{ lock_service::ObjectLockStatus, mutex_table::{LockGuard, MutexTable}, LockService, }; -use sui_types::crypto::AuthoritySignInfo; -use sui_types::message_envelope::VerifiedEnvelope; use sui_types::object::Owner; use sui_types::storage::ChildObjectResolver; use sui_types::{base_types::SequenceNumber, storage::ParentSync}; use sui_types::{batch::TxSequenceNumber, object::PACKAGE_VERSION}; +use tokio_retry::strategy::{jitter, ExponentialBackoff}; +use tracing::{debug, info, trace}; use typed_store::rocks::DBBatch; use typed_store::traits::Map; -use crate::authority::authority_per_epoch_store::{ - AuthorityPerEpochStore, ExecutionIndicesWithHash, -}; - -use super::{authority_store_tables::AuthorityPerpetualTables, *}; - -pub type AuthorityStore = SuiDataStore; - const NUM_SHARDS: usize = 4096; const SHARD_SIZE: usize = 128; diff --git a/crates/sui-core/src/unit_tests/authority_tests.rs b/crates/sui-core/src/unit_tests/authority_tests.rs index f3137f1443c0e..468e509f8ae7b 100644 --- a/crates/sui-core/src/unit_tests/authority_tests.rs +++ b/crates/sui-core/src/unit_tests/authority_tests.rs @@ -2681,7 +2681,7 @@ async fn test_store_get_dynamic_object() { let add_cert = init_certified_transaction(add_txn, &authority_state); let add_effects = authority_state - .handle_certificate(&add_cert) + .execute_certificate_internal(&add_cert) .await .unwrap() .signed_effects @@ -2754,7 +2754,7 @@ async fn test_store_get_dynamic_field() { let add_cert = init_certified_transaction(add_txn, &authority_state); let add_effects = authority_state - .handle_certificate(&add_cert) + .execute_certificate_internal(&add_cert) .await .unwrap() .signed_effects diff --git a/crates/sui-core/src/unit_tests/move_integration_tests.rs b/crates/sui-core/src/unit_tests/move_integration_tests.rs index 3872b2f16a18d..fb3d096cf260f 100644 --- a/crates/sui-core/src/unit_tests/move_integration_tests.rs +++ b/crates/sui-core/src/unit_tests/move_integration_tests.rs @@ -23,7 +23,6 @@ use sui_types::{ use std::path::PathBuf; use std::{env, str::FromStr}; -use sui_types::object::Owner; const MAX_GAS: u64 = 10000; diff --git a/crates/sui-json-rpc-types/src/lib.rs b/crates/sui-json-rpc-types/src/lib.rs index 0f4b041d461ab..150eb215a51f5 100644 --- a/crates/sui-json-rpc-types/src/lib.rs +++ b/crates/sui-json-rpc-types/src/lib.rs @@ -11,7 +11,6 @@ use std::fmt::{Display, Formatter}; use std::str::FromStr; use colored::Colorize; -use fastcrypto::encoding::{Base64, Encoding}; use itertools::Itertools; use move_binary_format::file_format::{Ability, AbilitySet, StructTypeParameter, Visibility}; use move_binary_format::normalized::{ @@ -29,6 +28,7 @@ use serde::Serialize; use serde_json::Value; use serde_with::serde_as; +use fastcrypto::encoding::{Base64, Encoding}; use sui_json::SuiJsonValue; use sui_types::base_types::{ AuthorityName, ObjectDigest, ObjectID, ObjectInfo, ObjectRef, SequenceNumber, SuiAddress, diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index 96e1b58fb0ec2..ae55fecbce9bd 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -1,23 +1,21 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; - -use fastcrypto::encoding::Base64; use jsonrpsee::core::RpcResult; use jsonrpsee_proc_macros::rpc; +use std::collections::BTreeMap; +use sui_types::sui_system_state::SuiSystemState; +use fastcrypto::encoding::Base64; use sui_json::SuiJsonValue; use sui_json_rpc_types::{ - Balance, CoinPage, EventPage, GetObjectDataResponse, GetPastObjectDataResponse, - GetRawObjectDataResponse, MoveFunctionArgType, RPCTransactionRequestParams, SuiCoinMetadata, - SuiEventEnvelope, SuiEventFilter, SuiExecuteTransactionResponse, SuiMoveNormalizedFunction, - DynamicFieldPage, EventPage, GetObjectDataResponse, GetPastObjectDataResponse, - GetRawObjectDataResponse, MoveFunctionArgType, RPCTransactionRequestParams, SuiCoinMetadata, - SuiEventEnvelope, SuiEventFilter, SuiExecuteTransactionResponse, SuiMoveNormalizedFunction, - SuiMoveNormalizedModule, SuiMoveNormalizedStruct, SuiObjectInfo, - SuiTransactionAuthSignersResponse, SuiTransactionEffects, SuiTransactionFilter, - SuiTransactionResponse, SuiTypeTag, TransactionBytes, TransactionsPage, + Balance, CoinPage, DynamicFieldPage, EventPage, GetObjectDataResponse, + GetPastObjectDataResponse, GetRawObjectDataResponse, MoveFunctionArgType, + RPCTransactionRequestParams, SuiCoinMetadata, SuiEventEnvelope, SuiEventFilter, + SuiExecuteTransactionResponse, SuiMoveNormalizedFunction, SuiMoveNormalizedModule, + SuiMoveNormalizedStruct, SuiObjectInfo, SuiTransactionAuthSignersResponse, + SuiTransactionEffects, SuiTransactionFilter, SuiTransactionResponse, SuiTypeTag, + TransactionBytes, TransactionsPage, }; use sui_open_rpc_macros::open_rpc; use sui_types::balance::Supply; @@ -29,7 +27,6 @@ use sui_types::event::EventID; use sui_types::messages::CommitteeInfoResponse; use sui_types::messages::ExecuteTransactionRequestType; use sui_types::query::{EventQuery, TransactionQuery}; -use sui_types::sui_system_state::SuiSystemState; /// Maximum number of events returned in an event query. /// This is equivalent to EVENT_QUERY_MAX_LIMIT in `sui-storage` crate. diff --git a/crates/sui-json-rpc/src/coin_api.rs b/crates/sui-json-rpc/src/coin_api.rs index e937d298ab1b6..9b8e1beaebc52 100644 --- a/crates/sui-json-rpc/src/coin_api.rs +++ b/crates/sui-json-rpc/src/coin_api.rs @@ -21,7 +21,7 @@ use sui_types::coin::{Coin, CoinMetadata, TreasuryCap}; use sui_types::error::SuiError; use sui_types::event::Event; use sui_types::gas_coin::GAS; -use sui_types::object::{Object, Owner}; +use sui_types::object::Object; use sui_types::parse_sui_struct_tag; use crate::api::{cap_page_limit, CoinReadApiServer}; @@ -63,7 +63,7 @@ impl CoinReadApi { ) -> Result + '_, Error> { Ok(self .state - .get_owner_objects_iterator(Owner::AddressOwner(owner))? + .get_owner_objects_iterator(owner)? .filter(move |o| matches!(&o.type_, ObjectType::Struct(type_) if is_coin_type(type_, coin_type))) .map(|info|info.object_id)) } @@ -119,7 +119,7 @@ impl CoinReadApiServer for CoinReadApi { limit: Option, ) -> RpcResult { // TODO: Add index to improve performance? - let limit = cap_page_limit(limit)?; + let limit = cap_page_limit(limit); let mut coins = self .get_owner_coin_iterator(owner, &coin_type)? .skip_while(|o| matches!(&cursor, Some(cursor) if cursor != o)) diff --git a/crates/sui-json-rpc/src/read_api.rs b/crates/sui-json-rpc/src/read_api.rs index d8ee7a6c7b7b9..da77830a2df71 100644 --- a/crates/sui-json-rpc/src/read_api.rs +++ b/crates/sui-json-rpc/src/read_api.rs @@ -6,20 +6,18 @@ use async_trait::async_trait; use jsonrpsee::core::RpcResult; use move_binary_format::normalized::{Module as NormalizedModule, Type}; use move_core_types::identifier::Identifier; -use move_core_types::language_storage::StructTag; use std::collections::BTreeMap; use std::sync::Arc; use sui_types::intent::{Intent, IntentMessage}; use sui_types::sui_system_state::SuiSystemState; use tap::TapFallible; -use tracing::debug; use fastcrypto::encoding::Base64; use jsonrpsee::RpcModule; use sui_core::authority::AuthorityState; use sui_json_rpc_types::{ DynamicFieldPage, GetObjectDataResponse, GetPastObjectDataResponse, MoveFunctionArgType, - ObjectValueKind, Page, SuiMoveNormalizedFunction, SuiMoveNormalizedModule, + ObjectValueKind, Page, SuiMoveNormalizedFunction, SuiMoveNormalizedModule, SuiMoveNormalizedStruct, SuiObjectInfo, SuiTransactionAuthSignersResponse, SuiTransactionEffects, SuiTransactionResponse, TransactionsPage, }; @@ -34,6 +32,8 @@ use sui_types::move_package::normalize_modules; use sui_types::object::{Data, ObjectRead}; use sui_types::query::TransactionQuery; +use tracing::debug; + use crate::api::RpcFullNodeReadApiServer; use crate::api::{cap_page_limit, RpcReadApiServer}; use crate::SuiRpcModule; diff --git a/crates/sui-open-rpc/spec/openrpc.json b/crates/sui-open-rpc/spec/openrpc.json index 41c1ebb761a56..2a64536af28cf 100644 --- a/crates/sui-open-rpc/spec/openrpc.json +++ b/crates/sui-open-rpc/spec/openrpc.json @@ -657,6 +657,82 @@ } } }, + { + "name": "sui_getDynamicFieldObject", + "tags": [ + { + "name": "Read API" + } + ], + "description": "Return the dynamic field object information for a specified object", + "params": [ + { + "name": "parent_object_id", + "description": "The ID of the queried parent object", + "required": true, + "schema": { + "$ref": "#/components/schemas/ObjectID" + } + }, + { + "name": "name", + "description": "The Name of the dynamic field", + "required": true, + "schema": { + "type": "string" + } + } + ], + "result": { + "name": "GetObjectDataResponse", + "required": true, + "schema": { + "$ref": "#/components/schemas/ObjectRead" + } + } + }, + { + "name": "sui_getDynamicFields", + "tags": [ + { + "name": "Read API" + } + ], + "description": "Return the list of dynamic field objects owned by an object.", + "params": [ + { + "name": "parent_object_id", + "description": "The ID of the parent object", + "required": true, + "schema": { + "$ref": "#/components/schemas/ObjectID" + } + }, + { + "name": "cursor", + "description": "Optional paging cursor", + "schema": { + "$ref": "#/components/schemas/ObjectID" + } + }, + { + "name": "limit", + "description": "Maximum item returned per page", + "schema": { + "type": "integer", + "format": "uint", + "minimum": 0.0 + } + } + ], + "result": { + "name": "DynamicFieldPage", + "required": true, + "schema": { + "$ref": "#/components/schemas/Page_for_DynamicFieldInfo_and_ObjectID" + } + } + }, { "name": "sui_getEvents", "tags": [ @@ -2638,6 +2714,67 @@ } ] }, + "DynamicFieldInfo": { + "type": "object", + "required": [ + "digest", + "name", + "objectId", + "objectType", + "type", + "version" + ], + "properties": { + "digest": { + "$ref": "#/components/schemas/ObjectDigest" + }, + "name": { + "type": "string" + }, + "objectId": { + "$ref": "#/components/schemas/ObjectID" + }, + "objectType": { + "type": "string" + }, + "type": { + "$ref": "#/components/schemas/DynamicFieldType" + }, + "version": { + "$ref": "#/components/schemas/SequenceNumber" + } + } + }, + "DynamicFieldType": { + "oneOf": [ + { + "type": "string", + "enum": [ + "DynamicObject" + ] + }, + { + "type": "object", + "required": [ + "DynamicField" + ], + "properties": { + "DynamicField": { + "type": "object", + "required": [ + "wrappedObjectId" + ], + "properties": { + "wrappedObjectId": { + "$ref": "#/components/schemas/ObjectID" + } + } + } + }, + "additionalProperties": false + } + ] + }, "Ed25519SuiSignature": { "$ref": "#/components/schemas/Base64" }, @@ -4039,6 +4176,30 @@ } } }, + "Page_for_DynamicFieldInfo_and_ObjectID": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DynamicFieldInfo" + } + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/ObjectID" + }, + { + "type": "null" + } + ] + } + } + }, "Page_for_EventEnvelope_and_EventID": { "type": "object", "required": [ diff --git a/crates/sui-storage/src/indexes.rs b/crates/sui-storage/src/indexes.rs index edc05d4c86910..ff59c29a3f0da 100644 --- a/crates/sui-storage/src/indexes.rs +++ b/crates/sui-storage/src/indexes.rs @@ -7,6 +7,11 @@ use move_core_types::identifier::Identifier; use serde::{de::DeserializeOwned, Serialize}; use tracing::debug; +use typed_store::rocks::DBMap; +use typed_store::rocks::DBOptions; +use typed_store::traits::Map; +use typed_store::traits::TypedStoreDebug; +use typed_store_derive::DBMapUtils; use sui_types::base_types::{ObjectID, SuiAddress, TransactionDigest}; use sui_types::base_types::{ObjectInfo, ObjectRef}; @@ -14,11 +19,6 @@ use sui_types::batch::TxSequenceNumber; use sui_types::dynamic_field::DynamicFieldInfo; use sui_types::error::SuiResult; use sui_types::object::Owner; -use typed_store::rocks::DBMap; -use typed_store::rocks::DBOptions; -use typed_store::traits::Map; -use typed_store::traits::TypedStoreDebug; -use typed_store_derive::DBMapUtils; use crate::default_db_options; @@ -380,15 +380,22 @@ impl IndexStore { } pub fn get_owner_objects(&self, owner: SuiAddress) -> SuiResult> { + debug!(?owner, "get_owner_objects"); + Ok(self.get_owner_objects_iterator(owner)?.collect()) + } + + pub fn get_owner_objects_iterator( + &self, + owner: SuiAddress, + ) -> SuiResult + '_> { debug!(?owner, "get_owner_objects"); Ok(self .owner_index .iter() // The object id 0 is the smallest possible .skip_to(&(owner, ObjectID::ZERO))? - .take_while(|((object_owner, _), _)| (object_owner == &owner)) - .map(|(_, object_info)| object_info) - .collect()) + .take_while(move |((object_owner, _), _)| (object_owner == &owner)) + .map(|(_, object_info)| object_info)) } pub fn insert_genesis_objects(&self, object_index_changes: ObjectIndexChanges) -> SuiResult { From 6013e6f44da699c84e7ccb1f1dd3ef9ff96bdde1 Mon Sep 17 00:00:00 2001 From: patrick Date: Tue, 13 Dec 2022 21:49:54 +0000 Subject: [PATCH 13/13] address PR comments --- crates/sui-json-rpc/src/api.rs | 6 +++--- crates/sui-open-rpc/spec/openrpc.json | 6 +++--- crates/sui-sdk/src/apis.rs | 21 +++++++++++++++++---- crates/sui-sdk/src/lib.rs | 6 +----- crates/sui-types/src/base_types.rs | 2 -- 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/crates/sui-json-rpc/src/api.rs b/crates/sui-json-rpc/src/api.rs index ae55fecbce9bd..b5abf3496aee9 100644 --- a/crates/sui-json-rpc/src/api.rs +++ b/crates/sui-json-rpc/src/api.rs @@ -105,7 +105,7 @@ pub trait RpcReadApi { parent_object_id: ObjectID, /// Optional paging cursor cursor: Option, - /// Maximum item returned per page + /// Maximum item returned per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified. limit: Option, ) -> RpcResult; @@ -215,7 +215,7 @@ pub trait RpcFullNodeReadApi { query: TransactionQuery, /// Optional paging cursor cursor: Option, - /// Maximum item returned per page + /// Maximum item returned per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified. limit: Option, /// query result ordering, default to false (ascending order), oldest record first. descending_order: Option, @@ -494,7 +494,7 @@ pub trait EventReadApi { query: EventQuery, /// optional paging cursor cursor: Option, - /// maximum number of items per page + /// maximum number of items per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified. limit: Option, /// query result ordering, default to false (ascending order), oldest record first. descending_order: Option, diff --git a/crates/sui-open-rpc/spec/openrpc.json b/crates/sui-open-rpc/spec/openrpc.json index 2a64536af28cf..96238075aea8d 100644 --- a/crates/sui-open-rpc/spec/openrpc.json +++ b/crates/sui-open-rpc/spec/openrpc.json @@ -717,7 +717,7 @@ }, { "name": "limit", - "description": "Maximum item returned per page", + "description": "Maximum item returned per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified.", "schema": { "type": "integer", "format": "uint", @@ -759,7 +759,7 @@ }, { "name": "limit", - "description": "maximum number of items per page", + "description": "maximum number of items per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified.", "schema": { "type": "integer", "format": "uint", @@ -1606,7 +1606,7 @@ }, { "name": "limit", - "description": "Maximum item returned per page", + "description": "Maximum item returned per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified.", "schema": { "type": "integer", "format": "uint", diff --git a/crates/sui-sdk/src/apis.rs b/crates/sui-sdk/src/apis.rs index 5fe7be7603be3..f1a8584f96221 100644 --- a/crates/sui-sdk/src/apis.rs +++ b/crates/sui-sdk/src/apis.rs @@ -10,10 +10,10 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::{Duration, Instant}; use sui_json_rpc_types::{ - Balance, Coin, CoinPage, EventPage, GetObjectDataResponse, GetPastObjectDataResponse, - GetRawObjectDataResponse, SuiCoinMetadata, SuiEventEnvelope, SuiEventFilter, - SuiExecuteTransactionResponse, SuiMoveNormalizedModule, SuiObjectInfo, SuiTransactionResponse, - TransactionsPage, + Balance, Coin, CoinPage, DynamicFieldPage, EventPage, GetObjectDataResponse, + GetPastObjectDataResponse, GetRawObjectDataResponse, SuiCoinMetadata, SuiEventEnvelope, + SuiEventFilter, SuiExecuteTransactionResponse, SuiMoveNormalizedModule, SuiObjectInfo, + SuiTransactionResponse, TransactionsPage, }; use sui_types::balance::Supply; use sui_types::base_types::{ObjectID, SequenceNumber, SuiAddress, TransactionDigest}; @@ -56,6 +56,19 @@ impl ReadApi { Ok(self.api.http.get_objects_owned_by_object(object_id).await?) } + pub async fn get_dynamic_fields( + &self, + object_id: ObjectID, + cursor: Option, + limit: Option, + ) -> SuiRpcResult { + Ok(self + .api + .http + .get_dynamic_fields(object_id, cursor, limit) + .await?) + } + pub async fn get_parsed_object( &self, object_id: ObjectID, diff --git a/crates/sui-sdk/src/lib.rs b/crates/sui-sdk/src/lib.rs index bd3aa06aced3d..65d73c938cc3d 100644 --- a/crates/sui-sdk/src/lib.rs +++ b/crates/sui-sdk/src/lib.rs @@ -19,11 +19,7 @@ pub use sui_json as json; use crate::apis::{CoinReadApi, EventApi, QuorumDriver, ReadApi}; pub use sui_json_rpc_types as rpc_types; -use sui_json_rpc_types::{ - DynamicFieldPage, EventPage, GetObjectDataResponse, GetRawObjectDataResponse, SuiEventEnvelope, - SuiEventFilter, SuiMoveNormalizedModule, SuiObjectInfo, SuiTransactionResponse, - TransactionsPage, -}; +use sui_json_rpc_types::{GetRawObjectDataResponse, SuiObjectInfo}; use sui_transaction_builder::{DataReader, TransactionBuilder}; pub use sui_types as types; use sui_types::base_types::{ObjectID, SuiAddress, TransactionDigest}; diff --git a/crates/sui-types/src/base_types.rs b/crates/sui-types/src/base_types.rs index 682c16055159a..cc723412d15bd 100644 --- a/crates/sui-types/src/base_types.rs +++ b/crates/sui-types/src/base_types.rs @@ -34,8 +34,6 @@ use crate::error::SuiError; use crate::gas_coin::GasCoin; use crate::object::{Object, Owner}; use crate::sui_serde::Readable; -use crate::waypoint::IntoPoint; -use crate::SUI_FRAMEWORK_ADDRESS; use fastcrypto::encoding::{Base58, Base64, Encoding, Hex}; use fastcrypto::hash::{HashFunction, Sha3_256};