From ec9f1c0035af5b980e5650f656793379ad9cc2bd Mon Sep 17 00:00:00 2001 From: Alexandcoats Date: Tue, 22 Nov 2022 06:32:39 -0500 Subject: [PATCH] feat(influxdb)!: consolidate queries (#921) * Consolidate leder_output_analytics queries * Consolidate output activity queries * Consolidate block activity analytics --- .../dashboards/analytics_dashboard.json | 663 +++++++++--------- src/db/collections/analytics/influx.rs | 92 +-- src/db/collections/analytics/mod.rs | 48 +- src/db/collections/block.rs | 45 +- src/db/collections/outputs/mod.rs | 166 ++--- tests/blocks.rs | 26 +- tests/outputs.rs | 90 +-- 7 files changed, 534 insertions(+), 596 deletions(-) diff --git a/docker/assets/grafana/dashboards/analytics_dashboard.json b/docker/assets/grafana/dashboards/analytics_dashboard.json index 926916de5..364b5ef13 100644 --- a/docker/assets/grafana/dashboards/analytics_dashboard.json +++ b/docker/assets/grafana/dashboards/analytics_dashboard.json @@ -137,7 +137,7 @@ } ], "hide": false, - "measurement": "stardust_payload_activity", + "measurement": "stardust_block_activity", "orderByTime": "ASC", "policy": "default", "refId": "B", @@ -321,7 +321,7 @@ "type": "fill" } ], - "measurement": "stardust_transaction_activity", + "measurement": "stardust_block_activity", "orderByTime": "ASC", "policy": "default", "refId": "A", @@ -1190,7 +1190,7 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -1198,358 +1198,357 @@ "y": 36 }, "id": 59, - "panels": [ - { - "datasource": { - "type": "influxdb", - "uid": "P951FEA4DE68E13C5" + "panels": [], + "title": "Output Activity", + "type": "row" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P951FEA4DE68E13C5" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "alias": "$col", - "datasource": { - "type": "influxdb", - "uid": "P951FEA4DE68E13C5" - }, - "groupBy": [ - { - "params": [ - "$aggregation_interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "stardust_alias_activity", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "created_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "Created" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "governor_changed_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "Governor Changed" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "state_changed_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "State Changed" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "destroyed_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "Destroyed" - ], - "type": "alias" - } - ] - ], - "tags": [] - } - ], - "title": "Alias Activity Counts", - "type": "timeseries" + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ { + "alias": "$col", "datasource": { "type": "influxdb", "uid": "P951FEA4DE68E13C5" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "groupBy": [ + { + "params": [ + "$aggregation_interval" + ], + "type": "time" }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 37 + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stardust_output_activity", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "alias_created_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Created" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "alias_governor_changed_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Governor Changed" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "alias_state_changed_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "State Changed" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "alias_destroyed_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Destroyed" + ], + "type": "alias" + } + ] + ], + "tags": [] + } + ], + "title": "Alias Activity Counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P951FEA4DE68E13C5" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 60, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "multi", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "alias": "$col", + "datasource": { + "type": "influxdb", + "uid": "P951FEA4DE68E13C5" + }, + "groupBy": [ { - "alias": "$col", - "datasource": { - "type": "influxdb", - "uid": "P951FEA4DE68E13C5" - }, - "groupBy": [ - { - "params": [ - "$aggregation_interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } + "params": [ + "$aggregation_interval" ], - "measurement": "stardust_nft_activity", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "created_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "Created" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "transferred_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "Transfered" - ], - "type": "alias" - } - ], - [ - { - "params": [ - "destroyed_count" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - }, - { - "params": [ - "Destroyed" - ], - "type": "alias" - } - ] + "type": "time" + }, + { + "params": [ + "null" ], - "tags": [] + "type": "fill" } ], - "title": "NFT Activity Counts", - "type": "timeseries" + "measurement": "stardust_output_activity", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "nft_created_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Created" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "nft_transferred_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Transfered" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "nft_destroyed_count" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Destroyed" + ], + "type": "alias" + } + ] + ], + "tags": [] } ], - "title": "Output Activity", - "type": "row" + "title": "NFT Activity Counts", + "type": "timeseries" }, { "collapsed": false, diff --git a/src/db/collections/analytics/influx.rs b/src/db/collections/analytics/influx.rs index 23f0034e5..0785c60e0 100644 --- a/src/db/collections/analytics/influx.rs +++ b/src/db/collections/analytics/influx.rs @@ -48,13 +48,11 @@ impl InfluxDb { self.insert_analytics(milestone_timestamp, milestone_index, analytics.addresses), self.insert_analytics(milestone_timestamp, milestone_index, analytics.base_token), self.insert_analytics(milestone_timestamp, milestone_index, analytics.ledger_outputs), - self.insert_analytics(milestone_timestamp, milestone_index, analytics.aliases), - self.insert_analytics(milestone_timestamp, milestone_index, analytics.nfts), - self.insert_analytics(milestone_timestamp, milestone_index, analytics.storage_deposits), + self.insert_analytics(milestone_timestamp, milestone_index, analytics.output_activity), + self.insert_analytics(milestone_timestamp, milestone_index, analytics.ledger_size), self.insert_analytics(milestone_timestamp, milestone_index, analytics.unclaimed_tokens), - self.insert_analytics(milestone_timestamp, milestone_index, analytics.payload_activity), + self.insert_analytics(milestone_timestamp, milestone_index, analytics.block_activity), self.insert_analytics(milestone_timestamp, milestone_index, analytics.unlock_conditions), - self.insert_analytics(milestone_timestamp, milestone_index, analytics.transaction_activity), async { if let Some(protocol_params) = analytics.protocol_params { self.insert_analytics(milestone_timestamp, milestone_index, protocol_params) @@ -184,36 +182,27 @@ impl InfluxDbMeasurement for AnalyticsSchema { const NAME: &'static str = "stardust_unclaimed_rewards"; } -impl InfluxDbWriteable for AnalyticsSchema { +impl InfluxDbWriteable for AnalyticsSchema { fn into_query>(self, name: I) -> influxdb::WriteQuery { Timestamp::from(self.milestone_timestamp) .into_query(name) .add_field("milestone_index", self.milestone_index) - .add_field("transaction_count", self.data.transaction_count) - .add_field("treasury_transaction_count", self.data.treasury_transaction_count) - .add_field("milestone_count", self.data.milestone_count) - .add_field("tagged_data_count", self.data.tagged_data_count) - .add_field("no_payload_count", self.data.no_payload_count) - } -} - -impl InfluxDbMeasurement for AnalyticsSchema { - const NAME: &'static str = "stardust_payload_activity"; -} - -impl InfluxDbWriteable for AnalyticsSchema { - fn into_query>(self, name: I) -> influxdb::WriteQuery { - Timestamp::from(self.milestone_timestamp) - .into_query(name) - .add_field("milestone_index", self.milestone_index) - .add_field("confirmed_count", self.data.confirmed_count) - .add_field("conflicting_count", self.data.conflicting_count) - .add_field("no_transaction_count", self.data.no_transaction_count) + .add_field("transaction_count", self.data.payload.transaction_count) + .add_field( + "treasury_transaction_count", + self.data.payload.treasury_transaction_count, + ) + .add_field("milestone_count", self.data.payload.milestone_count) + .add_field("tagged_data_count", self.data.payload.tagged_data_count) + .add_field("no_payload_count", self.data.payload.no_payload_count) + .add_field("confirmed_count", self.data.transaction.confirmed_count) + .add_field("conflicting_count", self.data.transaction.conflicting_count) + .add_field("no_transaction_count", self.data.transaction.no_transaction_count) } } -impl InfluxDbMeasurement for AnalyticsSchema { - const NAME: &'static str = "stardust_transaction_activity"; +impl InfluxDbMeasurement for AnalyticsSchema { + const NAME: &'static str = "stardust_block_activity"; } impl InfluxDbWriteable for AnalyticsSchema { @@ -234,50 +223,23 @@ impl InfluxDbMeasurement for AnalyticsSchema { const NAME: &'static str = "stardust_protocol_params"; } -impl InfluxDbWriteable for AnalyticsSchema { - fn into_query>(self, name: I) -> influxdb::WriteQuery { - Timestamp::from(self.milestone_timestamp) - .into_query(name) - .add_field("milestone_index", self.milestone_index) - .add_field("created_count", self.data.created_count) - .add_field("state_changed_count", self.data.state_changed_count) - .add_field("governor_changed_count", self.data.governor_changed_count) - .add_field("destroyed_count", self.data.destroyed_count) - } -} - -impl InfluxDbMeasurement for AnalyticsSchema { - const NAME: &'static str = "stardust_alias_activity"; -} - -impl InfluxDbWriteable for AnalyticsSchema { - fn into_query>(self, name: I) -> influxdb::WriteQuery { - Timestamp::from(self.milestone_timestamp) - .into_query(name) - .add_field("milestone_index", self.milestone_index) - .add_field("created_count", self.data.created_count) - .add_field("transferred_count", self.data.transferred_count) - .add_field("destroyed_count", self.data.destroyed_count) - } -} - -impl InfluxDbMeasurement for AnalyticsSchema { - const NAME: &'static str = "stardust_foundry_activity"; -} - -impl InfluxDbWriteable for AnalyticsSchema { +impl InfluxDbWriteable for AnalyticsSchema { fn into_query>(self, name: I) -> influxdb::WriteQuery { Timestamp::from(self.milestone_timestamp) .into_query(name) .add_field("milestone_index", self.milestone_index) - .add_field("created_count", self.data.created_count) - .add_field("transferred_count", self.data.transferred_count) - .add_field("destroyed_count", self.data.destroyed_count) + .add_field("alias_created_count", self.data.alias.created_count) + .add_field("alias_state_changed_count", self.data.alias.state_changed_count) + .add_field("alias_governor_changed_count", self.data.alias.governor_changed_count) + .add_field("alias_destroyed_count", self.data.alias.destroyed_count) + .add_field("nft_created_count", self.data.nft.created_count) + .add_field("nft_transferred_count", self.data.nft.transferred_count) + .add_field("nft_destroyed_count", self.data.nft.destroyed_count) } } -impl InfluxDbMeasurement for AnalyticsSchema { - const NAME: &'static str = "stardust_nft_activity"; +impl InfluxDbMeasurement for AnalyticsSchema { + const NAME: &'static str = "stardust_output_activity"; } impl InfluxDbWriteable for AnalyticsSchema { diff --git a/src/db/collections/analytics/mod.rs b/src/db/collections/analytics/mod.rs index bc0301363..01a72ee8e 100644 --- a/src/db/collections/analytics/mod.rs +++ b/src/db/collections/analytics/mod.rs @@ -23,12 +23,10 @@ pub struct Analytics { pub addresses: AddressAnalytics, pub base_token: BaseTokenActivityAnalytics, pub ledger_outputs: LedgerOutputAnalytics, - pub aliases: AliasActivityAnalytics, - pub nfts: NftActivityAnalytics, - pub storage_deposits: LedgerSizeAnalytics, + pub output_activity: OutputActivityAnalytics, + pub ledger_size: LedgerSizeAnalytics, pub unclaimed_tokens: UnclaimedTokensAnalytics, - pub payload_activity: PayloadActivityAnalytics, - pub transaction_activity: TransactionActivityAnalytics, + pub block_activity: BlockActivityAnalytics, pub unlock_conditions: UnlockConditionAnalytics, pub protocol_params: Option, } @@ -44,28 +42,24 @@ impl MongoDb { let ( addresses, ledger_outputs, - aliases, - nfts, - storage_deposits, + output_activity, + ledger_size, unclaimed_tokens, unlock_conditions, address_activity, base_token, - payload_activity, - transaction_activity, + block_activity, protocol_params, ) = tokio::try_join!( output_collection.get_address_analytics(milestone_index), output_collection.get_ledger_output_analytics(milestone_index), - output_collection.get_alias_output_analytics(milestone_index), - output_collection.get_nft_output_analytics(milestone_index), + output_collection.get_output_activity_analytics(milestone_index), output_collection.get_ledger_size_analytics(milestone_index), output_collection.get_unclaimed_token_analytics(milestone_index), output_collection.get_unlock_condition_analytics(milestone_index), output_collection.get_address_activity_analytics(milestone_index), output_collection.get_base_token_activity_analytics(milestone_index), - block_collection.get_payload_activity_analytics(milestone_index), - block_collection.get_transaction_activity_analytics(milestone_index), + block_collection.get_block_activity_analytics(milestone_index), protocol_param_collection .get_protocol_parameters_for_milestone_index(milestone_index) .and_then(|p| async move { Ok(p.map(|p| p.parameters)) }), @@ -76,12 +70,10 @@ impl MongoDb { addresses, base_token, ledger_outputs, - aliases, - nfts, - storage_deposits, + output_activity, + ledger_size, unclaimed_tokens, - payload_activity, - transaction_activity, + block_activity, unlock_conditions, protocol_params, }) @@ -158,6 +150,15 @@ pub struct UnclaimedTokensAnalytics { #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[allow(missing_docs)] +#[serde(default)] +pub struct OutputActivityAnalytics { + pub alias: AliasActivityAnalytics, + pub nft: NftActivityAnalytics, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[allow(missing_docs)] +#[serde(default)] pub struct AliasActivityAnalytics { pub created_count: u64, pub governor_changed_count: u64, @@ -167,6 +168,7 @@ pub struct AliasActivityAnalytics { #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[allow(missing_docs)] +#[serde(default)] pub struct NftActivityAnalytics { pub created_count: u64, pub transferred_count: u64, @@ -181,12 +183,20 @@ pub struct BaseTokenActivityAnalytics { #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[allow(missing_docs)] +#[serde(default)] pub struct FoundryActivityAnalytics { pub created_count: u64, pub transferred_count: u64, pub destroyed_count: u64, } +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub struct BlockActivityAnalytics { + pub payload: PayloadActivityAnalytics, + pub transaction: TransactionActivityAnalytics, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[allow(missing_docs)] pub struct PayloadActivityAnalytics { diff --git a/src/db/collections/block.rs b/src/db/collections/block.rs index f50afdd6e..2f71eb1ad 100644 --- a/src/db/collections/block.rs +++ b/src/db/collections/block.rs @@ -316,18 +316,15 @@ impl BlockCollection { #[cfg(feature = "analytics")] mod analytics { use super::*; - use crate::{ - db::collections::analytics::{PayloadActivityAnalytics, TransactionActivityAnalytics}, - types::tangle::MilestoneIndex, - }; + use crate::{db::collections::analytics::BlockActivityAnalytics, types::tangle::MilestoneIndex}; impl BlockCollection { /// Gathers past-cone payload activity statistics for a given milestone. #[tracing::instrument(skip(self), err, level = "trace")] - pub async fn get_payload_activity_analytics( + pub async fn get_block_activity_analytics( &self, index: MilestoneIndex, - ) -> Result { + ) -> Result { Ok(self .aggregate( vec![ @@ -349,28 +346,6 @@ mod analytics { "no_payload_count": { "$sum": { "$cond": [ { "$not": "$block.payload" }, 1 , 0 ] } }, - } }, - ], - None, - ) - .await? - .try_next() - .await? - .unwrap_or_default()) - } - - /// Gathers past-cone transaction activity statistics for a given milestone. - #[tracing::instrument(skip(self), err, level = "trace")] - pub async fn get_transaction_activity_analytics( - &self, - index: MilestoneIndex, - ) -> Result { - Ok(self - .aggregate( - vec![ - doc! { "$match": { "metadata.referenced_by_milestone_index": index } }, - doc! { "$group": { - "_id": null, "confirmed_count": { "$sum": { "$cond": [ { "$eq": [ "$metadata.inclusion_state", "included" ] }, 1 , 0 ] } }, @@ -381,6 +356,20 @@ mod analytics { "$cond": [ { "$eq": [ "$metadata.inclusion_state", "no_transaction" ] }, 1 , 0 ] } }, } }, + doc! { "$project": { + "payload": { + "transaction_count": "$transaction_count", + "treasury_transaction_count": "$treasury_transaction_count", + "milestone_count": "$milestone_count", + "tagged_data_count": "$tagged_data_count", + "no_payload_count": "$no_payload_count", + }, + "transaction": { + "confirmed_count": "$confirmed_count", + "conflicting_count": "$conflicting_count", + "no_transaction_count": "$no_transaction_count", + } + } }, ], None, ) diff --git a/src/db/collections/outputs/mod.rs b/src/db/collections/outputs/mod.rs index 25c4c85af..a11ad96de 100644 --- a/src/db/collections/outputs/mod.rs +++ b/src/db/collections/outputs/mod.rs @@ -390,16 +390,13 @@ mod analytics { use crate::{ db::{ collections::analytics::{ - AddressActivityAnalytics, AddressAnalytics, AliasActivityAnalytics, BaseTokenActivityAnalytics, - LedgerOutputAnalytics, LedgerSizeAnalytics, NftActivityAnalytics, UnclaimedTokensAnalytics, - UnlockConditionAnalytics, + AddressActivityAnalytics, AddressAnalytics, BaseTokenActivityAnalytics, LedgerOutputAnalytics, + LedgerSizeAnalytics, OutputActivityAnalytics, UnclaimedTokensAnalytics, UnlockConditionAnalytics, }, mongodb::MongoDbCollectionExt, }, types::{ - stardust::block::output::{ - AliasId, AliasOutput, BasicOutput, FoundryOutput, NftId, NftOutput, TreasuryOutput, - }, + stardust::block::output::{AliasId, NftId}, tangle::MilestoneIndex, }, }; @@ -445,76 +442,85 @@ mod analytics { ledger_index: MilestoneIndex, ) -> Result { #[derive(Default, Deserialize)] - struct Res { + struct Sums { count: u64, value: d128, } - let query = |kind: &'static str| async move { - Result::<_, Error>::Ok( - self.aggregate::( - vec![ - doc! { "$match": { - "output.kind": kind, - "metadata.booked.milestone_index": { "$lte": ledger_index }, - "metadata.spent_metadata.spent.milestone_index": { "$not": { "$lte": ledger_index } } - } }, - doc! { "$group" : { - "_id": null, - "count": { "$sum": 1 }, - "value": { "$sum": { "$toDecimal": "$output.amount" } }, - } }, - doc! { "$project": { - "count": 1, - "value": { "$toString": "$value" }, + #[derive(Default, Deserialize)] + #[serde(default)] + struct Res { + basic: Sums, + alias: Sums, + foundry: Sums, + nft: Sums, + treasury: Sums, + } + + let res = self + .aggregate::( + vec![ + doc! { "$match": { + "metadata.booked.milestone_index": { "$lte": ledger_index }, + "metadata.spent_metadata.spent.milestone_index": { "$not": { "$lte": ledger_index } } + } }, + doc! { "$group" : { + "_id": "$output.kind", + "count": { "$sum": 1 }, + "value": { "$sum": { "$toDecimal": "$output.amount" } }, + } }, + doc! { "$group" : { + "_id": null, + "result": { "$addToSet": { + "k": "$_id", + "v": { + "count": "$count", + "value": { "$toString": "$value" }, + } } }, - ], - None, - ) - .await? - .try_next() - .await? - .unwrap_or_default(), + } }, + doc! { "$replaceWith": { + "$arrayToObject": "$result" + } }, + ], + None, ) - }; - - let (basic, alias, foundry, nft, treasury) = tokio::try_join!( - query(BasicOutput::KIND), - query(AliasOutput::KIND), - query(FoundryOutput::KIND), - query(NftOutput::KIND), - query(TreasuryOutput::KIND) - )?; + .await? + .try_next() + .await? + .unwrap_or_default(); Ok(crate::db::collections::analytics::LedgerOutputAnalytics { - basic_count: basic.count, - basic_value: basic.value, - alias_count: alias.count, - alias_value: alias.value, - foundry_count: foundry.count, - foundry_value: foundry.value, - nft_count: nft.count, - nft_value: nft.value, - treasury_count: treasury.count, - treasury_value: treasury.value, + basic_count: res.basic.count, + basic_value: res.basic.value, + alias_count: res.alias.count, + alias_value: res.alias.value, + foundry_count: res.foundry.count, + foundry_value: res.foundry.value, + nft_count: res.nft.count, + nft_value: res.nft.value, + treasury_count: res.treasury.count, + treasury_value: res.treasury.value, }) } - /// Gathers analytics about nfts that were created/transferred/burned in the given milestone. + /// Gathers analytics about outputs that were created/transferred/burned in the given milestone. #[tracing::instrument(skip(self), err, level = "trace")] - pub async fn get_nft_output_analytics(&self, index: MilestoneIndex) -> Result { + pub async fn get_output_activity_analytics( + &self, + index: MilestoneIndex, + ) -> Result { Ok(self .aggregate( vec![ doc! { "$match": { - "output.kind": "nft", "$or": [ { "metadata.booked.milestone_index": index }, { "metadata.spent_metadata.spent.milestone_index": index }, ], } }, doc! { "$facet": { - "created": [ + "nft_created": [ { "$match": { "metadata.booked.milestone_index": index, "output.nft_id": NftId::implicit(), @@ -524,7 +530,7 @@ mod analytics { "count": { "$sum": 1 }, } }, ], - "changed": [ + "nft_changed": [ { "$match": { "output.nft_id": { "$ne": NftId::implicit() } } }, { "$group": { "_id": "$output.nft_id", @@ -537,36 +543,7 @@ mod analytics { "destroyed": { "$sum": { "$cond": [ { "$eq": [ "$unspent", 0 ] }, 1, 0 ] } }, } }, ], - } }, - doc! { "$project": { - "created_count": { "$ifNull": [ { "$first": "$created.count" }, 0 ] }, - "transferred_count": { "$ifNull": [ { "$first": "$changed.transferred" }, 0 ] }, - "destroyed_count": { "$ifNull": [ { "$first": "$changed.destroyed" }, 0 ] }, - } }, - ], - None, - ) - .await? - .try_next() - .await? - .unwrap_or_default()) - } - - /// Gathers analytics about aliases that were created/transferred/burned in the given milestone. - #[tracing::instrument(skip(self), err, level = "trace")] - pub async fn get_alias_output_analytics(&self, index: MilestoneIndex) -> Result { - Ok(self - .aggregate( - vec![ - doc! { "$match": { - "output.kind": "alias", - "$or": [ - { "metadata.booked.milestone_index": index }, - { "metadata.spent_metadata.spent.milestone_index": index }, - ], - } }, - doc! { "$facet": { - "created": [ + "alias_created": [ { "$match": { "metadata.booked.milestone_index": index, "output.alias_id": AliasId::implicit(), @@ -576,7 +553,7 @@ mod analytics { "count": { "$sum": 1 }, } }, ], - "changed": [ + "alias_changed": [ { "$match": { "output.alias_id": { "$ne": AliasId::implicit() } } }, // Group by state indexes to find where it changed { "$group": { @@ -601,11 +578,18 @@ mod analytics { ], } }, doc! { "$project": { - "created_count": { "$ifNull": [ { "$first": "$created.count" }, 0 ] }, - "state_changed_count": { "$ifNull": [ { "$first": "$changed.state" }, 0 ] }, - "governor_changed_count": { "$ifNull": [ { "$first": "$changed.governor" }, 0 ] }, - "destroyed_count": { "$ifNull": [ { "$first": "$changed.destroyed" }, 0 ] }, - } } + "alias": { + "created_count": { "$first": "$alias_created.count" }, + "state_changed_count": { "$first": "$alias_changed.state" }, + "governor_changed_count": { "$first": "$alias_changed.governor" }, + "destroyed_count": { "$first": "$alias_changed.destroyed" }, + }, + "nft": { + "created_count": { "$first": "$nft_created.count" }, + "transferred_count": { "$first": "$nft_changed.transferred" }, + "destroyed_count": { "$first": "$nft_changed.destroyed" }, + }, + } }, ], None, ) diff --git a/tests/blocks.rs b/tests/blocks.rs index f01183e0c..59779defe 100644 --- a/tests/blocks.rs +++ b/tests/blocks.rs @@ -324,22 +324,16 @@ mod test_rand { .await .unwrap(); - let activity = block_collection.get_payload_activity_analytics(1.into()).await.unwrap(); - - assert_eq!(activity.transaction_count, 1); - assert_eq!(activity.treasury_transaction_count, 1); - assert_eq!(activity.milestone_count, 1); - assert_eq!(activity.tagged_data_count, 1); - assert_eq!(activity.no_payload_count, 1); - - let activity = block_collection - .get_transaction_activity_analytics(1.into()) - .await - .unwrap(); - - assert_eq!(activity.confirmed_count, 1); - assert_eq!(activity.conflicting_count, 1); - assert_eq!(activity.no_transaction_count, 3); + let activity = block_collection.get_block_activity_analytics(1.into()).await.unwrap(); + + assert_eq!(activity.payload.transaction_count, 1); + assert_eq!(activity.payload.treasury_transaction_count, 1); + assert_eq!(activity.payload.milestone_count, 1); + assert_eq!(activity.payload.tagged_data_count, 1); + assert_eq!(activity.payload.no_payload_count, 1); + assert_eq!(activity.transaction.confirmed_count, 1); + assert_eq!(activity.transaction.conflicting_count, 1); + assert_eq!(activity.transaction.no_transaction_count, 3); teardown(db).await; } diff --git a/tests/outputs.rs b/tests/outputs.rs index 7644a53bf..8a459e2ce 100644 --- a/tests/outputs.rs +++ b/tests/outputs.rs @@ -234,12 +234,12 @@ mod test_rand { .collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_alias_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 1); - assert_eq!(analytics.governor_changed_count, 1); - assert_eq!(analytics.state_changed_count, 2); - assert_eq!(analytics.destroyed_count, 0); + assert_eq!(analytics.alias.created_count, 1); + assert_eq!(analytics.alias.governor_changed_count, 1); + assert_eq!(analytics.alias.state_changed_count, 2); + assert_eq!(analytics.alias.destroyed_count, 0); // t -> s -> s let mut output = AliasOutput::rand(&protocol_params); @@ -281,12 +281,12 @@ mod test_rand { .collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_alias_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 1); - assert_eq!(analytics.governor_changed_count, 2); - assert_eq!(analytics.state_changed_count, 4); - assert_eq!(analytics.destroyed_count, 0); + assert_eq!(analytics.alias.created_count, 1); + assert_eq!(analytics.alias.governor_changed_count, 2); + assert_eq!(analytics.alias.state_changed_count, 4); + assert_eq!(analytics.alias.destroyed_count, 0); // s -> t -> d let mut output = AliasOutput::rand(&protocol_params); @@ -319,12 +319,12 @@ mod test_rand { let consumed_outputs = created_outputs.into_iter().map(ledger_spent).collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_alias_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 1); - assert_eq!(analytics.governor_changed_count, 3); - assert_eq!(analytics.state_changed_count, 5); - assert_eq!(analytics.destroyed_count, 1); + assert_eq!(analytics.alias.created_count, 1); + assert_eq!(analytics.alias.governor_changed_count, 3); + assert_eq!(analytics.alias.state_changed_count, 5); + assert_eq!(analytics.alias.destroyed_count, 1); // c -> s -> s -> d let mut output = AliasOutput::rand(&protocol_params); @@ -342,12 +342,12 @@ mod test_rand { let consumed_outputs = created_outputs.into_iter().map(ledger_spent).collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_alias_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 2); - assert_eq!(analytics.governor_changed_count, 3); - assert_eq!(analytics.state_changed_count, 7); - assert_eq!(analytics.destroyed_count, 2); + assert_eq!(analytics.alias.created_count, 2); + assert_eq!(analytics.alias.governor_changed_count, 3); + assert_eq!(analytics.alias.state_changed_count, 7); + assert_eq!(analytics.alias.destroyed_count, 2); // c -> t -> t -> d let mut output = AliasOutput::rand(&protocol_params); @@ -369,12 +369,12 @@ mod test_rand { let consumed_outputs = created_outputs.into_iter().map(ledger_spent).collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_alias_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 3); - assert_eq!(analytics.governor_changed_count, 5); - assert_eq!(analytics.state_changed_count, 7); - assert_eq!(analytics.destroyed_count, 3); + assert_eq!(analytics.alias.created_count, 3); + assert_eq!(analytics.alias.governor_changed_count, 5); + assert_eq!(analytics.alias.state_changed_count, 7); + assert_eq!(analytics.alias.destroyed_count, 3); teardown(db).await; } @@ -430,11 +430,11 @@ mod test_rand { .collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_nft_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 1); - assert_eq!(analytics.transferred_count, 3); - assert_eq!(analytics.destroyed_count, 0); + assert_eq!(analytics.nft.created_count, 1); + assert_eq!(analytics.nft.transferred_count, 3); + assert_eq!(analytics.nft.destroyed_count, 0); // t -> t -> t let output = NftOutput::rand(&protocol_params); @@ -472,11 +472,11 @@ mod test_rand { .collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_nft_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 1); - assert_eq!(analytics.transferred_count, 6); - assert_eq!(analytics.destroyed_count, 0); + assert_eq!(analytics.nft.created_count, 1); + assert_eq!(analytics.nft.transferred_count, 6); + assert_eq!(analytics.nft.destroyed_count, 0); // t -> t -> d let output = NftOutput::rand(&protocol_params); @@ -504,11 +504,11 @@ mod test_rand { let consumed_outputs = created_outputs.into_iter().map(ledger_spent).collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_nft_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 1); - assert_eq!(analytics.transferred_count, 8); - assert_eq!(analytics.destroyed_count, 1); + assert_eq!(analytics.nft.created_count, 1); + assert_eq!(analytics.nft.transferred_count, 8); + assert_eq!(analytics.nft.destroyed_count, 1); // c -> t -> t -> d let output = NftOutput::rand(&protocol_params); @@ -526,11 +526,11 @@ mod test_rand { let consumed_outputs = created_outputs.into_iter().map(ledger_spent).collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_nft_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 2); - assert_eq!(analytics.transferred_count, 10); - assert_eq!(analytics.destroyed_count, 2); + assert_eq!(analytics.nft.created_count, 2); + assert_eq!(analytics.nft.transferred_count, 10); + assert_eq!(analytics.nft.destroyed_count, 2); // c -> t -> t -> d let output = NftOutput::rand(&protocol_params); @@ -548,11 +548,11 @@ mod test_rand { let consumed_outputs = created_outputs.into_iter().map(ledger_spent).collect::>(); output_collection.update_spent_outputs(&consumed_outputs).await.unwrap(); - let analytics = output_collection.get_nft_output_analytics(2.into()).await.unwrap(); + let analytics = output_collection.get_output_activity_analytics(2.into()).await.unwrap(); - assert_eq!(analytics.created_count, 3); - assert_eq!(analytics.transferred_count, 12); - assert_eq!(analytics.destroyed_count, 3); + assert_eq!(analytics.nft.created_count, 3); + assert_eq!(analytics.nft.transferred_count, 12); + assert_eq!(analytics.nft.destroyed_count, 3); teardown(db).await; }