From b3b51317dce8bf5e637d52db2fd9a8a0dd7b3021 Mon Sep 17 00:00:00 2001 From: stefan-mysten <135084671+stefan-mysten@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:04:22 -0800 Subject: [PATCH] [GraphQL] Add a new `multiGetObjects` query on `Query`. (#20300) ## Description This PR adds a new query for fetching multiple objects by their ids and versions. ## Test plan Added two new tests to check that the limits work as expected. - `test_multi_get_objects_query_limits_pass` - `test_multi_get_objects_query_limits_exceeded` --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [x] GraphQL: added a `multiGetObjects` top level query, which will replace the objectFilter in `objects`. - [ ] CLI: - [ ] Rust SDK: --- crates/sui-graphql-rpc/schema.graphql | 12 +- crates/sui-graphql-rpc/src/config.rs | 14 ++- .../src/extensions/query_limits_checker.rs | 47 ++++++- crates/sui-graphql-rpc/src/server/builder.rs | 119 ++++++++++++++++++ crates/sui-graphql-rpc/src/types/object.rs | 38 +++++- crates/sui-graphql-rpc/src/types/query.rs | 21 ++++ crates/sui-graphql-rpc/staging.graphql | 12 +- .../snapshot_tests__schema.graphql.snap | 12 +- .../snapshot_tests__staging.graphql.snap | 12 +- crates/sui-mvr-graphql-rpc/schema.graphql | 12 +- crates/sui-mvr-graphql-rpc/src/config.rs | 12 ++ .../src/extensions/query_limits_checker.rs | 47 ++++++- .../sui-mvr-graphql-rpc/src/server/builder.rs | 119 ++++++++++++++++++ .../sui-mvr-graphql-rpc/src/types/object.rs | 38 +++++- crates/sui-mvr-graphql-rpc/src/types/query.rs | 21 ++++ crates/sui-mvr-graphql-rpc/staging.graphql | 12 +- .../snapshot_tests__schema.graphql.snap | 14 ++- .../snapshot_tests__staging.graphql.snap | 14 ++- 18 files changed, 553 insertions(+), 23 deletions(-) diff --git a/crates/sui-graphql-rpc/schema.graphql b/crates/sui-graphql-rpc/schema.graphql index bdeadd932bc75..cfd9ede2416ea 100644 --- a/crates/sui-graphql-rpc/schema.graphql +++ b/crates/sui-graphql-rpc/schema.graphql @@ -2899,7 +2899,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3325,6 +3327,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3599,6 +3605,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-graphql-rpc/src/config.rs b/crates/sui-graphql-rpc/src/config.rs index c2664d97842e7..974ce37350856 100644 --- a/crates/sui-graphql-rpc/src/config.rs +++ b/crates/sui-graphql-rpc/src/config.rs @@ -118,10 +118,12 @@ pub struct Limits { pub max_type_argument_width: u32, /// Maximum size of a fully qualified type. pub max_type_nodes: u32, - /// Maximum deph of a move value. + /// Maximum depth of a move value. pub max_move_value_depth: u32, /// Maximum number of transaction ids that can be passed to a `TransactionBlockFilter`. pub max_transaction_ids: u32, + /// Maximum number of keys that can be passed to a `multiGetObjects` query. + pub max_multi_get_objects_keys: u32, /// Maximum number of candidates to scan when gathering a page of results. pub max_scan_limit: u32, } @@ -343,6 +345,11 @@ impl ServiceConfig { self.limits.max_transaction_ids } + /// Maximum number of keys that can be passed to a `multiGetObjects` query. + async fn max_multi_get_objects_keys(&self) -> u32 { + self.limits.max_multi_get_objects_keys + } + /// Maximum number of candidates to scan when gathering a page of results. async fn max_scan_limit(&self) -> u32 { self.limits.max_scan_limit @@ -510,6 +517,7 @@ impl Default for Limits { // Filter-specific limits, such as the number of transaction ids that can be specified // for the `TransactionBlockFilter`. max_transaction_ids: 1000, + max_multi_get_objects_keys: 500, max_scan_limit: 100_000_000, // This value is set to be the size of the max transaction bytes allowed + base64 // overhead (roughly 1/3 of the original string). This is rounded up. @@ -594,6 +602,7 @@ mod tests { max-type-nodes = 128 max-move-value-depth = 256 max-transaction-ids = 11 + max-multi-get-objects-keys = 11 max-scan-limit = 50 "#, ) @@ -616,6 +625,7 @@ mod tests { max_type_nodes: 128, max_move_value_depth: 256, max_transaction_ids: 11, + max_multi_get_objects_keys: 11, max_scan_limit: 50, }, ..Default::default() @@ -682,6 +692,7 @@ mod tests { max-type-nodes = 128 max-move-value-depth = 256 max-transaction-ids = 42 + max-multi-get-objects-keys = 42 max-scan-limit = 420 [experiments] @@ -707,6 +718,7 @@ mod tests { max_type_nodes: 128, max_move_value_depth: 256, max_transaction_ids: 42, + max_multi_get_objects_keys: 42, max_scan_limit: 420, }, disabled_features: BTreeSet::from([FunctionalGroup::Analytics]), diff --git a/crates/sui-graphql-rpc/src/extensions/query_limits_checker.rs b/crates/sui-graphql-rpc/src/extensions/query_limits_checker.rs index 458a513a8046f..976f46a62d202 100644 --- a/crates/sui-graphql-rpc/src/extensions/query_limits_checker.rs +++ b/crates/sui-graphql-rpc/src/extensions/query_limits_checker.rs @@ -29,6 +29,8 @@ use uuid::Uuid; pub(crate) const CONNECTION_FIELDS: [&str; 2] = ["edges", "nodes"]; const DRY_RUN_TX_BLOCK: &str = "dryRunTransactionBlock"; const EXECUTE_TX_BLOCK: &str = "executeTransactionBlock"; +const MULTI_GET_PREFIX: &str = "multiGet"; +const MULTI_GET_KEYS: &str = "keys"; const VERIFY_ZKLOGIN: &str = "verifyZkloginSignature"; /// The size of the query payload in bytes, as it comes from the request header: `Content-Length`. @@ -228,6 +230,7 @@ impl<'a> LimitsTraversal<'a> { match &item.node { Selection::Field(f) => { let name = &f.node.name.node; + if name == DRY_RUN_TX_BLOCK || name == EXECUTE_TX_BLOCK { for (_name, value) in &f.node.arguments { self.check_tx_arg(value)?; @@ -415,12 +418,15 @@ impl<'a> LimitsTraversal<'a> { self.output_budget -= multiplicity; } - // If the field being traversed is a connection field, increase multiplicity by a - // factor of page size. This operation can fail due to overflow, which will be - // treated as a limits check failure, even if the resulting value does not get used - // for anything. let name = &f.node.name.node; + + // Handle regular connection fields and multiGet queries let multiplicity = 'm: { + // check if it is a multiGet query and return the number of keys + if let Some(page_size) = self.multi_get_page_size(f)? { + break 'm multiplicity * page_size; + } + if !CONNECTION_FIELDS.contains(&name.as_str()) { break 'm multiplicity; } @@ -428,7 +434,6 @@ impl<'a> LimitsTraversal<'a> { let Some(page_size) = page_size else { break 'm multiplicity; }; - multiplicity .checked_mul(page_size) .ok_or_else(|| self.output_node_error())? @@ -484,6 +489,23 @@ impl<'a> LimitsTraversal<'a> { )) } + // If the field `f` is a multiGet query, extract the number of keys, otherwise return `None`. + // Returns an error if the number of keys cannot be represented as a `u32`. + fn multi_get_page_size(&mut self, f: &Positioned) -> ServerResult> { + if !f.node.name.node.starts_with(MULTI_GET_PREFIX) { + return Ok(None); + } + + let keys = f.node.get_argument(MULTI_GET_KEYS); + let Some(page_size) = self.resolve_list_size(keys) else { + return Ok(None); + }; + + Ok(Some( + page_size.try_into().map_err(|_| self.output_node_error())?, + )) + } + /// Checks if the given field corresponds to a connection based on whether it contains a /// selection for `edges` or `nodes`. That selection could be immediately in that field's /// selection set, or nested within a fragment or inline fragment spread. @@ -548,6 +570,21 @@ impl<'a> LimitsTraversal<'a> { .as_u64() } + /// Find the size of a list, resolving variables if necessary. + fn resolve_list_size(&self, value: Option<&Positioned>) -> Option { + match &value?.node { + Value::List(list) => Some(list.len()), + Value::Variable(var) => { + if let ConstValue::List(list) = self.variables.get(var)? { + Some(list.len()) + } else { + None + } + } + _ => None, + } + } + /// Error returned if transaction payloads exceed limit. Also sets the transaction payload /// budget to zero to indicate it has been spent (This is done to prevent future checks for /// smaller arguments from succeeding even though a previous larger argument has already diff --git a/crates/sui-graphql-rpc/src/server/builder.rs b/crates/sui-graphql-rpc/src/server/builder.rs index d674ddc38f490..8a061e11665cf 100644 --- a/crates/sui-graphql-rpc/src/server/builder.rs +++ b/crates/sui-graphql-rpc/src/server/builder.rs @@ -2067,4 +2067,123 @@ pub mod tests { bytes or fewer." ); } + + #[tokio::test] + async fn test_multi_get_objects_query_limits_exceeded() { + let cluster = prep_executor_cluster().await; + let db_url = cluster.graphql_connection_config.db_url.clone(); + assert_eq!( + execute_for_error( + &db_url, + Limits { + max_output_nodes: 5, + ..Default::default() + }, // the query will have 6 output nodes: 2 keys * 3 fields, thus exceeding the + // limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + ] + ) { + address + status + version + } + } + "# + .into(), + ) + .await, + "Estimated output nodes exceeds 5" + ); + assert_eq!( + execute_for_error( + &db_url, + Limits { + max_output_nodes: 4, + ..Default::default() + }, // the query will have 5 output nodes: 5keys * 1 field, thus exceeding the limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x33032e0706337632361f2607b79df8c9d1079e8069259b27b1fa5c0394e79893", version: 2}, + {objectId: "0x388295e3ecad53986ebf9a7a1e5854b7df94c3f1f0bba934c5396a2a9eb4550b", version: 2}, + ] + ) { + address + } + } + "# + .into(), + ) + .await, + "Estimated output nodes exceeds 4" + ); + } + + #[tokio::test] + async fn test_multi_get_objects_query_limits_pass() { + let cluster = prep_executor_cluster().await; + let db_url = cluster.graphql_connection_config.db_url.clone(); + let service_config = ServiceConfig { + limits: Limits { + max_output_nodes: 5, + ..Default::default() + }, + ..Default::default() + }; + + let schema = prep_schema(db_url, Some(service_config)) + .await + .build_schema(); + + let resp = schema + .execute( + // query will have 5 output nodes: 5 keys * 1 field, thus not exceeding the limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x33032e0706337632361f2607b79df8c9d1079e8069259b27b1fa5c0394e79893", version: 2}, + {objectId: "0x388295e3ecad53986ebf9a7a1e5854b7df94c3f1f0bba934c5396a2a9eb4550b", version: 2}, + ] + ) { + address + } + } + "#) + .await; + assert!(resp.is_ok()); + assert!(resp.errors.is_empty()); + + let resp = schema + .execute( + // query will have 4 output nodes: 2 keys * 2 fields, thus not exceeding the limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + ] + ) { + address + status + } + } + "#) + .await; + assert!(resp.is_ok()); + assert!(resp.errors.is_empty()); + } } diff --git a/crates/sui-graphql-rpc/src/types/object.rs b/crates/sui-graphql-rpc/src/types/object.rs index 918ec71e979ff..c450dd77f291a 100644 --- a/crates/sui-graphql-rpc/src/types/object.rs +++ b/crates/sui-graphql-rpc/src/types/object.rs @@ -130,7 +130,9 @@ pub(crate) struct ObjectFilter { /// Filter for live objects by their IDs. pub object_ids: Option>, - /// Filter for live or potentially historical objects by their ID and version. + /// Filter for live objects by their ID and version. NOTE: this input filter has been + /// deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + /// objects by their versions. This filter will be removed with v1.42.0 release. pub object_keys: Option>, } @@ -801,6 +803,40 @@ impl Object { self.root_version } + /// Fetch objects by their id and version. If you need to query for live objects, use the + /// `objects` field. + pub(crate) async fn query_many( + ctx: &Context<'_>, + keys: Vec, + checkpoint_viewed_at: u64, + ) -> Result, Error> { + let DataLoader(loader) = &ctx.data_unchecked(); + + let keys: Vec = keys + .into_iter() + .map(|key| PointLookupKey { + id: key.object_id, + version: key.version.into(), + }) + .collect(); + + let data = loader.load_many(keys).await?; + let objects: Vec<_> = data + .into_iter() + .filter_map(|(lookup_key, bcs)| { + Object::new_serialized( + lookup_key.id, + lookup_key.version, + bcs, + checkpoint_viewed_at, + lookup_key.version, + ) + }) + .collect(); + + Ok(objects) + } + /// Query the database for a `page` of objects, optionally `filter`-ed. /// /// `checkpoint_viewed_at` represents the checkpoint sequence number at which this page was diff --git a/crates/sui-graphql-rpc/src/types/query.rs b/crates/sui-graphql-rpc/src/types/query.rs index d9077b352408f..101a4e38c58a6 100644 --- a/crates/sui-graphql-rpc/src/types/query.rs +++ b/crates/sui-graphql-rpc/src/types/query.rs @@ -17,6 +17,7 @@ use super::move_package::{ }; use super::move_registry::named_move_package::NamedMovePackage; use super::move_registry::named_type::NamedType; +use super::object::ObjectKey; use super::suins_registration::NameService; use super::uint53::UInt53; use super::{ @@ -327,6 +328,26 @@ impl Query { TransactionBlock::query(ctx, lookup).await.extend() } + /// Fetch a list of objects by their IDs and versions. + async fn multi_get_objects( + &self, + ctx: &Context<'_>, + keys: Vec, + ) -> Result> { + let cfg: &ServiceConfig = ctx.data_unchecked(); + if keys.len() > cfg.limits.max_multi_get_objects_keys as usize { + return Err(Error::Client(format!( + "Number of keys exceeds max limit of '{}'", + cfg.limits.max_multi_get_objects_keys + )) + .into()); + } + + let Watermark { hi_cp, .. } = *ctx.data()?; + + Object::query_many(ctx, keys, hi_cp).await.extend() + } + /// The coin objects that exist in the network. /// /// The type field is a string of the inner type of the coin by which to filter (e.g. diff --git a/crates/sui-graphql-rpc/staging.graphql b/crates/sui-graphql-rpc/staging.graphql index 96e2db10710bb..3abd78866738c 100644 --- a/crates/sui-graphql-rpc/staging.graphql +++ b/crates/sui-graphql-rpc/staging.graphql @@ -2899,7 +2899,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3325,6 +3327,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3599,6 +3605,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap index 51fb4649ffe7f..ad94721c71818 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap @@ -2903,7 +2903,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3329,6 +3331,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3603,6 +3609,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap index a855e3cf80f34..3f2db2a8c601c 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap @@ -2903,7 +2903,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3329,6 +3331,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3603,6 +3609,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-mvr-graphql-rpc/schema.graphql b/crates/sui-mvr-graphql-rpc/schema.graphql index 92330bc77ffa1..3a369fb19b424 100644 --- a/crates/sui-mvr-graphql-rpc/schema.graphql +++ b/crates/sui-mvr-graphql-rpc/schema.graphql @@ -2880,7 +2880,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3306,6 +3308,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3579,6 +3585,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-mvr-graphql-rpc/src/config.rs b/crates/sui-mvr-graphql-rpc/src/config.rs index c2664d97842e7..571f426586ca6 100644 --- a/crates/sui-mvr-graphql-rpc/src/config.rs +++ b/crates/sui-mvr-graphql-rpc/src/config.rs @@ -122,6 +122,8 @@ pub struct Limits { pub max_move_value_depth: u32, /// Maximum number of transaction ids that can be passed to a `TransactionBlockFilter`. pub max_transaction_ids: u32, + /// Maximum number of keys that can be passed to a `multiGetObjects` query. + pub max_multi_get_objects_keys: u32, /// Maximum number of candidates to scan when gathering a page of results. pub max_scan_limit: u32, } @@ -343,6 +345,11 @@ impl ServiceConfig { self.limits.max_transaction_ids } + /// Maximum number of keys that can be passed to a `multiGetObjects` query. + async fn max_multi_get_objects_keys(&self) -> u32 { + self.limits.max_multi_get_objects_keys + } + /// Maximum number of candidates to scan when gathering a page of results. async fn max_scan_limit(&self) -> u32 { self.limits.max_scan_limit @@ -510,6 +517,7 @@ impl Default for Limits { // Filter-specific limits, such as the number of transaction ids that can be specified // for the `TransactionBlockFilter`. max_transaction_ids: 1000, + max_multi_get_objects_keys: 500, max_scan_limit: 100_000_000, // This value is set to be the size of the max transaction bytes allowed + base64 // overhead (roughly 1/3 of the original string). This is rounded up. @@ -594,6 +602,7 @@ mod tests { max-type-nodes = 128 max-move-value-depth = 256 max-transaction-ids = 11 + max-multi-get-objects-keys = 11 max-scan-limit = 50 "#, ) @@ -616,6 +625,7 @@ mod tests { max_type_nodes: 128, max_move_value_depth: 256, max_transaction_ids: 11, + max_multi_get_objects_keys: 11, max_scan_limit: 50, }, ..Default::default() @@ -682,6 +692,7 @@ mod tests { max-type-nodes = 128 max-move-value-depth = 256 max-transaction-ids = 42 + max-multi-get-objects-keys = 42 max-scan-limit = 420 [experiments] @@ -707,6 +718,7 @@ mod tests { max_type_nodes: 128, max_move_value_depth: 256, max_transaction_ids: 42, + max_multi_get_objects_keys: 42, max_scan_limit: 420, }, disabled_features: BTreeSet::from([FunctionalGroup::Analytics]), diff --git a/crates/sui-mvr-graphql-rpc/src/extensions/query_limits_checker.rs b/crates/sui-mvr-graphql-rpc/src/extensions/query_limits_checker.rs index 458a513a8046f..976f46a62d202 100644 --- a/crates/sui-mvr-graphql-rpc/src/extensions/query_limits_checker.rs +++ b/crates/sui-mvr-graphql-rpc/src/extensions/query_limits_checker.rs @@ -29,6 +29,8 @@ use uuid::Uuid; pub(crate) const CONNECTION_FIELDS: [&str; 2] = ["edges", "nodes"]; const DRY_RUN_TX_BLOCK: &str = "dryRunTransactionBlock"; const EXECUTE_TX_BLOCK: &str = "executeTransactionBlock"; +const MULTI_GET_PREFIX: &str = "multiGet"; +const MULTI_GET_KEYS: &str = "keys"; const VERIFY_ZKLOGIN: &str = "verifyZkloginSignature"; /// The size of the query payload in bytes, as it comes from the request header: `Content-Length`. @@ -228,6 +230,7 @@ impl<'a> LimitsTraversal<'a> { match &item.node { Selection::Field(f) => { let name = &f.node.name.node; + if name == DRY_RUN_TX_BLOCK || name == EXECUTE_TX_BLOCK { for (_name, value) in &f.node.arguments { self.check_tx_arg(value)?; @@ -415,12 +418,15 @@ impl<'a> LimitsTraversal<'a> { self.output_budget -= multiplicity; } - // If the field being traversed is a connection field, increase multiplicity by a - // factor of page size. This operation can fail due to overflow, which will be - // treated as a limits check failure, even if the resulting value does not get used - // for anything. let name = &f.node.name.node; + + // Handle regular connection fields and multiGet queries let multiplicity = 'm: { + // check if it is a multiGet query and return the number of keys + if let Some(page_size) = self.multi_get_page_size(f)? { + break 'm multiplicity * page_size; + } + if !CONNECTION_FIELDS.contains(&name.as_str()) { break 'm multiplicity; } @@ -428,7 +434,6 @@ impl<'a> LimitsTraversal<'a> { let Some(page_size) = page_size else { break 'm multiplicity; }; - multiplicity .checked_mul(page_size) .ok_or_else(|| self.output_node_error())? @@ -484,6 +489,23 @@ impl<'a> LimitsTraversal<'a> { )) } + // If the field `f` is a multiGet query, extract the number of keys, otherwise return `None`. + // Returns an error if the number of keys cannot be represented as a `u32`. + fn multi_get_page_size(&mut self, f: &Positioned) -> ServerResult> { + if !f.node.name.node.starts_with(MULTI_GET_PREFIX) { + return Ok(None); + } + + let keys = f.node.get_argument(MULTI_GET_KEYS); + let Some(page_size) = self.resolve_list_size(keys) else { + return Ok(None); + }; + + Ok(Some( + page_size.try_into().map_err(|_| self.output_node_error())?, + )) + } + /// Checks if the given field corresponds to a connection based on whether it contains a /// selection for `edges` or `nodes`. That selection could be immediately in that field's /// selection set, or nested within a fragment or inline fragment spread. @@ -548,6 +570,21 @@ impl<'a> LimitsTraversal<'a> { .as_u64() } + /// Find the size of a list, resolving variables if necessary. + fn resolve_list_size(&self, value: Option<&Positioned>) -> Option { + match &value?.node { + Value::List(list) => Some(list.len()), + Value::Variable(var) => { + if let ConstValue::List(list) = self.variables.get(var)? { + Some(list.len()) + } else { + None + } + } + _ => None, + } + } + /// Error returned if transaction payloads exceed limit. Also sets the transaction payload /// budget to zero to indicate it has been spent (This is done to prevent future checks for /// smaller arguments from succeeding even though a previous larger argument has already diff --git a/crates/sui-mvr-graphql-rpc/src/server/builder.rs b/crates/sui-mvr-graphql-rpc/src/server/builder.rs index 592a4a364520a..35a85d16eb134 100644 --- a/crates/sui-mvr-graphql-rpc/src/server/builder.rs +++ b/crates/sui-mvr-graphql-rpc/src/server/builder.rs @@ -2065,4 +2065,123 @@ pub mod tests { bytes or fewer." ); } + + #[tokio::test] + async fn test_multi_get_objects_query_limits_exceeded() { + let cluster = prep_executor_cluster().await; + let db_url = cluster.graphql_connection_config.db_url.clone(); + assert_eq!( + execute_for_error( + &db_url, + Limits { + max_output_nodes: 5, + ..Default::default() + }, // the query will have 6 output nodes: 2 keys * 3 fields, thus exceeding the + // limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + ] + ) { + address + status + version + } + } + "# + .into(), + ) + .await, + "Estimated output nodes exceeds 5" + ); + assert_eq!( + execute_for_error( + &db_url, + Limits { + max_output_nodes: 4, + ..Default::default() + }, // the query will have 5 output nodes, thus exceeding the limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x33032e0706337632361f2607b79df8c9d1079e8069259b27b1fa5c0394e79893", version: 2}, + {objectId: "0x388295e3ecad53986ebf9a7a1e5854b7df94c3f1f0bba934c5396a2a9eb4550b", version: 2}, + ] + ) { + address + } + } + "# + .into(), + ) + .await, + "Estimated output nodes exceeds 4" + ); + } + + #[tokio::test] + async fn test_multi_get_objects_query_limits_pass() { + let cluster = prep_executor_cluster().await; + let db_url = cluster.graphql_connection_config.db_url.clone(); + let service_config = ServiceConfig { + limits: Limits { + max_output_nodes: 5, + ..Default::default() + }, + ..Default::default() + }; + + let schema = prep_schema(db_url, Some(service_config)) + .await + .build_schema(); + + let resp = schema + .execute( + // query will have 5 output nodes: 5 keys * 1 field, thus not exceeding the limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + {objectId: "0x33032e0706337632361f2607b79df8c9d1079e8069259b27b1fa5c0394e79893", version: 2}, + {objectId: "0x388295e3ecad53986ebf9a7a1e5854b7df94c3f1f0bba934c5396a2a9eb4550b", version: 2}, + ] + ) { + address + } + } + "#) + .await; + assert!(resp.is_ok()); + assert!(resp.errors.is_empty()); + + let resp = schema + .execute( + // query will have 4 output nodes: 2 keys * 2 fields, thus not exceeding the limit + r#" + query { + multiGetObjects( + keys: [ + {objectId: "0x01dcb4674affb04e68d8088895e951f4ea335ef1695e9e50c166618f6789d808", version: 2}, + {objectId: "0x23e340e97fb41249278c85b1f067dc88576f750670c6dc56572e90971f857c8c", version: 2}, + ] + ) { + address + status + } + } + "#) + .await; + assert!(resp.is_ok()); + assert!(resp.errors.is_empty()); + } } diff --git a/crates/sui-mvr-graphql-rpc/src/types/object.rs b/crates/sui-mvr-graphql-rpc/src/types/object.rs index 59830f3f9b7be..c25bcc9cfd15a 100644 --- a/crates/sui-mvr-graphql-rpc/src/types/object.rs +++ b/crates/sui-mvr-graphql-rpc/src/types/object.rs @@ -129,7 +129,9 @@ pub(crate) struct ObjectFilter { /// Filter for live objects by their IDs. pub object_ids: Option>, - /// Filter for live or potentially historical objects by their ID and version. + /// Filter for live objects by their ID and version. NOTE: this input filter has been + /// deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + /// objects by their versions. This filter will be removed with v1.42.0 release. pub object_keys: Option>, } @@ -782,6 +784,40 @@ impl Object { self.root_version } + /// Fetch objects by their id and version. If you need to query for live objects, use the + /// `objects` field. + pub(crate) async fn query_many( + ctx: &Context<'_>, + keys: Vec, + checkpoint_viewed_at: u64, + ) -> Result, Error> { + let DataLoader(loader) = &ctx.data_unchecked(); + + let keys: Vec = keys + .into_iter() + .map(|key| PointLookupKey { + id: key.object_id, + version: key.version.into(), + }) + .collect(); + + let data = loader.load_many(keys).await?; + let objects: Vec<_> = data + .into_iter() + .filter_map(|(lookup_key, bcs)| { + Object::new_serialized( + lookup_key.id, + lookup_key.version, + bcs, + checkpoint_viewed_at, + lookup_key.version, + ) + }) + .collect(); + + Ok(objects) + } + /// Query the database for a `page` of objects, optionally `filter`-ed. /// /// `checkpoint_viewed_at` represents the checkpoint sequence number at which this page was diff --git a/crates/sui-mvr-graphql-rpc/src/types/query.rs b/crates/sui-mvr-graphql-rpc/src/types/query.rs index 74d17363a0c31..856632c54a6ad 100644 --- a/crates/sui-mvr-graphql-rpc/src/types/query.rs +++ b/crates/sui-mvr-graphql-rpc/src/types/query.rs @@ -44,6 +44,7 @@ use super::{ use crate::connection::ScanConnection; use crate::server::watermark_task::Watermark; use crate::types::base64::Base64 as GraphQLBase64; +use crate::types::object::ObjectKey; use crate::types::zklogin_verify_signature::verify_zklogin_signature; use crate::types::zklogin_verify_signature::ZkLoginIntentScope; use crate::types::zklogin_verify_signature::ZkLoginVerifyResult; @@ -327,6 +328,26 @@ impl Query { TransactionBlock::query(ctx, lookup).await.extend() } + /// Fetch a list of objects by their IDs and versions. + async fn multi_get_objects( + &self, + ctx: &Context<'_>, + keys: Vec, + ) -> Result> { + let cfg: &ServiceConfig = ctx.data_unchecked(); + if keys.len() > cfg.limits.max_multi_get_objects_keys as usize { + return Err(Error::Client(format!( + "Number of keys exceeds max limit of '{}'", + cfg.limits.max_multi_get_objects_keys + )) + .into()); + } + + let Watermark { checkpoint, .. } = *ctx.data()?; + + Object::query_many(ctx, keys, checkpoint).await.extend() + } + /// The coin objects that exist in the network. /// /// The type field is a string of the inner type of the coin by which to filter (e.g. diff --git a/crates/sui-mvr-graphql-rpc/staging.graphql b/crates/sui-mvr-graphql-rpc/staging.graphql index d37152f34e1bb..51190ef9b6daa 100644 --- a/crates/sui-mvr-graphql-rpc/staging.graphql +++ b/crates/sui-mvr-graphql-rpc/staging.graphql @@ -2880,7 +2880,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3306,6 +3308,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3579,6 +3585,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap b/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap index 70071a6173061..517ac38137cc4 100644 --- a/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap +++ b/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__schema.graphql.snap @@ -1,5 +1,5 @@ --- -source: crates/sui-graphql-rpc/tests/snapshot_tests.rs +source: crates/sui-mvr-graphql-rpc/tests/snapshot_tests.rs expression: sdl --- type ActiveJwk { @@ -2884,7 +2884,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3310,6 +3312,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3583,6 +3589,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int! diff --git a/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap b/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap index 498feb7c2fafb..b9daaa7fcf60b 100644 --- a/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap +++ b/crates/sui-mvr-graphql-rpc/tests/snapshots/snapshot_tests__staging.graphql.snap @@ -1,5 +1,5 @@ --- -source: crates/sui-graphql-rpc/tests/snapshot_tests.rs +source: crates/sui-mvr-graphql-rpc/tests/snapshot_tests.rs expression: sdl --- type ActiveJwk { @@ -2884,7 +2884,9 @@ input ObjectFilter { """ objectIds: [SuiAddress!] """ - Filter for live or potentially historical objects by their ID and version. + Filter for live objects by their ID and version. NOTE: this input filter has been + deprecated in favor of `multiGetObjects` query as it does not make sense to query for live + objects by their versions. This filter will be removed with v1.42.0 release. """ objectKeys: [ObjectKey!] } @@ -3310,6 +3312,10 @@ type Query { """ transactionBlock(digest: String!): TransactionBlock """ + Fetch a list of objects by their IDs and versions. + """ + multiGetObjects(keys: [ObjectKey!]!): [Object!]! + """ The coin objects that exist in the network. The type field is a string of the inner type of the coin by which to filter (e.g. @@ -3583,6 +3589,10 @@ type ServiceConfig { """ maxTransactionIds: Int! """ + Maximum number of keys that can be passed to a `multiGetObjects` query. + """ + maxMultiGetObjectsKeys: Int! + """ Maximum number of candidates to scan when gathering a page of results. """ maxScanLimit: Int!