From ae4777423486caf2027d008a0e7f8ef0587f9691 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 14 Mar 2023 17:49:41 +0100 Subject: [PATCH 01/58] checkpoint --- manta-accounting/src/wallet/signer/mod.rs | 39 +++++++++++++++++++++++ manta-pay/src/signer/base.rs | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index ed1055eb0..35515fe9d 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -108,6 +108,45 @@ where TransferPost: Clone; } +/// Signer Initial Synchronization Data +#[cfg_attr( + feature = "serde", + derive(Deserialize, Serialize), + serde( + bound( + deserialize = r" + Utxo: Deserialize<'de>, + UtxoMembershipProof: Deserialize<'de> + ", + serialize = r" + Utxo: Serialize, + UtxoMembershipProof: Serialize + ", + ), + crate = "manta_util::serde", + deny_unknown_fields + ) +)] +#[derive(derivative::Derivative)] +#[derivative( + Clone(bound = "Utxo: Clone, UtxoMembershipProof: Clone"), + Debug(bound = "Utxo: Debug, UtxoMembershipProof: Debug"), + Default(bound = ""), + Eq(bound = "Utxo: Eq, UtxoMembershipProof: Eq"), + Hash(bound = "Utxo: Hash, UtxoMembershipProof: Hash"), + PartialEq(bound = "Utxo: PartialEq, UtxoMembershipProof: PartialEq") +)] +pub struct InitialSyncData +where + C: transfer::Configuration + ?Sized, +{ + /// UTXO Data + pub utxo_data: Vec>, + + /// Membership Proof Data + pub membership_proof_data: manta_util::Array, NUMBER_OF_PROOFS>, +} + /// Signer Synchronization Data #[cfg_attr( feature = "serde", diff --git a/manta-pay/src/signer/base.rs b/manta-pay/src/signer/base.rs index b4c3c8ef2..ab0635a22 100644 --- a/manta-pay/src/signer/base.rs +++ b/manta-pay/src/signer/base.rs @@ -79,7 +79,7 @@ pub type UtxoAccumulator = merkle_tree::forest::TreeArrayMerkleForest< MerkleTreeConfiguration, merkle_tree::fork::ForkedTree< MerkleTreeConfiguration, - merkle_tree::full::Full, + merkle_tree::partial::Partial, >, { MerkleTreeConfiguration::FOREST_WIDTH }, >; From df6edd41ccb728121228982b0c88aa95f807fc3a Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 11:48:08 +0100 Subject: [PATCH 02/58] checkpoint --- .../src/wallet/signer/functions.rs | 65 ++++++++++++++++++- manta-accounting/src/wallet/signer/mod.rs | 22 ++++--- 2 files changed, 76 insertions(+), 11 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index bbc474f53..f2bfb0f6b 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -35,9 +35,9 @@ use crate::{ UtxoAccumulatorItem, UtxoAccumulatorModel, }, wallet::signer::{ - AccountTable, BalanceUpdate, Checkpoint, Configuration, SignError, SignResponse, - SignWithTransactionDataResponse, SignWithTransactionDataResult, SignerParameters, SyncData, - SyncError, SyncRequest, SyncResponse, + AccountTable, BalanceUpdate, Checkpoint, Configuration, InitialSyncData, SignError, + SignResponse, SignWithTransactionDataResponse, SignWithTransactionDataResult, + SignerParameters, SyncData, SyncError, SyncRequest, SyncResponse, }, }; use alloc::{vec, vec::Vec}; @@ -933,3 +933,62 @@ where .collect(), )) } + +/// Updates `assets`, `checkpoint` and `utxo_accumulator`, returning the new asset distribution. +#[inline] +pub fn intial_sync( + parameters: &SignerParameters, + authorization_context: &mut AuthorizationContext, + assets: &mut C::AssetMap, + checkpoint: &mut C::Checkpoint, + utxo_accumulator: &mut C::UtxoAccumulator, + request: InitialSyncData, + rng: &mut C::Rng, +) -> Result, SyncError> +where + C: Configuration, +{ + let InitialSyncData { + utxo_data, + membership_proof_data, + nullifier_data, + } = request; + let response = initial_sync_with::( + authorization_context, + assets, + checkpoint, + utxo_accumulator, + ¶meters.parameters, + utxo_data, + nullifier_data, + rng, + ); + utxo_accumulator.commit(); + Ok(response) +} + +/// Updates the internal ledger state, returning the new asset distribution. +#[allow(clippy::too_many_arguments)] +#[inline] +fn initial_sync_with( + authorization_context: &mut AuthorizationContext, + assets: &mut C::AssetMap, + checkpoint: &mut C::Checkpoint, + utxo_accumulator: &mut C::UtxoAccumulator, + parameters: &Parameters, + utxos: Vec>, + nullifiers: Vec>, + rng: &mut C::Rng, +) -> SyncResponse +where + C: Configuration, +{ + checkpoint.update_from_nullifiers(nullifiers.len()); + checkpoint.update_from_utxo_accumulator(utxo_accumulator); + SyncResponse { + checkpoint: checkpoint.clone(), + balance_update: BalanceUpdate::Full { + assets: assets.assets().into(), + }, + } +} diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 35515fe9d..466158470 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -116,11 +116,13 @@ where bound( deserialize = r" Utxo: Deserialize<'de>, - UtxoMembershipProof: Deserialize<'de> + UtxoMembershipProof: Deserialize<'de>, + Nullifier: Deserialize<'de>, ", serialize = r" Utxo: Serialize, - UtxoMembershipProof: Serialize + UtxoMembershipProof: Serialize, + Nullifier: Serialize, ", ), crate = "manta_util::serde", @@ -129,12 +131,13 @@ where )] #[derive(derivative::Derivative)] #[derivative( - Clone(bound = "Utxo: Clone, UtxoMembershipProof: Clone"), - Debug(bound = "Utxo: Debug, UtxoMembershipProof: Debug"), - Default(bound = ""), - Eq(bound = "Utxo: Eq, UtxoMembershipProof: Eq"), - Hash(bound = "Utxo: Hash, UtxoMembershipProof: Hash"), - PartialEq(bound = "Utxo: PartialEq, UtxoMembershipProof: PartialEq") + Clone(bound = "Utxo: Clone, UtxoMembershipProof: Clone, Nullifier: Clone"), + Debug(bound = "Utxo: Debug, UtxoMembershipProof: Debug, Nullifier: Debug"), + Eq(bound = "Utxo: Eq, UtxoMembershipProof: Eq, Nullifier: Eq"), + Hash(bound = "Utxo: Hash, UtxoMembershipProof: Hash, Nullifier: Hash"), + PartialEq( + bound = "Utxo: PartialEq, UtxoMembershipProof: PartialEq, Nullifier: PartialEq" + ) )] pub struct InitialSyncData where @@ -143,6 +146,9 @@ where /// UTXO Data pub utxo_data: Vec>, + /// Nullifier Data + pub nullifier_data: Vec>, + /// Membership Proof Data pub membership_proof_data: manta_util::Array, NUMBER_OF_PROOFS>, } From 164057a33be83fcf158cd7917cf576e767e99b88 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 13:32:04 +0100 Subject: [PATCH 03/58] checkpoint --- manta-accounting/src/wallet/signer/functions.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index f2bfb0f6b..fd909aea0 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -32,7 +32,7 @@ use crate::{ Address, Asset, AssociatedData, Authorization, AuthorizationContext, FullParametersRef, IdentifiedAsset, Identifier, IdentityProof, Note, Nullifier, Parameters, PreSender, ProvingContext, Receiver, Sender, Shape, SpendingKey, Transfer, TransferPost, Utxo, - UtxoAccumulatorItem, UtxoAccumulatorModel, + UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoMembershipProof, }, wallet::signer::{ AccountTable, BalanceUpdate, Checkpoint, Configuration, InitialSyncData, SignError, @@ -46,6 +46,7 @@ use manta_crypto::{ rand::Rand, }; use manta_util::{ + Array, array_map, cmp::Independence, into_array_unchecked, iter::IteratorExt, persistence::Rollback, vec::VecExt, }; @@ -953,13 +954,14 @@ where membership_proof_data, nullifier_data, } = request; - let response = initial_sync_with::( + let response = initial_sync_with::( authorization_context, assets, checkpoint, utxo_accumulator, ¶meters.parameters, utxo_data, + membership_proof_data, nullifier_data, rng, ); @@ -970,13 +972,14 @@ where /// Updates the internal ledger state, returning the new asset distribution. #[allow(clippy::too_many_arguments)] #[inline] -fn initial_sync_with( +fn initial_sync_with( authorization_context: &mut AuthorizationContext, assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator: &mut C::UtxoAccumulator, parameters: &Parameters, utxos: Vec>, + membership_proof_data: Array, NUMBER_OF_PROOFS>, nullifiers: Vec>, rng: &mut C::Rng, ) -> SyncResponse From 60c5eb8c4018275c9246ef13798f691c253335d4 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 17:27:50 +0100 Subject: [PATCH 04/58] merkle tree functions --- .../src/wallet/signer/functions.rs | 3 +- manta-crypto/src/merkle_tree/forest.rs | 65 +++++++++++++++++++ manta-crypto/src/merkle_tree/fork.rs | 21 ++++++ manta-crypto/src/merkle_tree/partial.rs | 32 +++++++++ 4 files changed, 119 insertions(+), 2 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index fd909aea0..8d30ebc93 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -46,9 +46,8 @@ use manta_crypto::{ rand::Rand, }; use manta_util::{ - Array, array_map, cmp::Independence, into_array_unchecked, iter::IteratorExt, persistence::Rollback, - vec::VecExt, + vec::VecExt, Array, }; /// Returns the default account for `accounts`. diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 959278dd8..2f2a6a166 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -30,6 +30,7 @@ use crate::{ merkle_tree::{ fork::ForkedTree, inner_tree::InnerMap, + partial::Partial, path::Path, tree::{self, Leaf, Parameters, Root, Tree}, InnerDigest, LeafDigest, WithProofs, @@ -525,6 +526,70 @@ where } } +impl TreeArray, N> +where + C: Configuration + ?Sized, + C::Index: FixedIndex, + LeafDigest: Clone + Default, + InnerDigest: Clone + Default + PartialEq, +{ + /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that + /// the `paths` are consistent with the leaves and that they are + /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. + #[inline] + pub fn from_leaves_and_current_paths_unchecked( + parameters: &Parameters, + leaves: Vec>, + paths: BoxArray, N>, + ) -> Self { + TreeArray::new(BoxArray::from_iter(paths.into_iter().enumerate().map( + |(tree_index, path)| { + Partial::from_leaves_and_path_unchecked( + parameters, + leaves + .iter() + .filter(|leaf| C::tree_index(leaf).into() == tree_index) + .map(|leaf| parameters.digest(leaf)) + .collect(), + path, + ) + }, + ))) + } +} + +impl TreeArray>, N> +where + C: Configuration + ?Sized, + C::Index: FixedIndex, + LeafDigest: Clone + Default, + InnerDigest: Clone + Default + PartialEq, +{ + /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that + /// the `paths` are consistent with the leaves and that they are + /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. + #[inline] + pub fn from_leaves_and_current_paths_unchecked( + parameters: &Parameters, + leaves: Vec>, + paths: BoxArray, N>, + ) -> Self { + TreeArray::new(BoxArray::from_iter(paths.into_iter().enumerate().map( + |(tree_index, path)| { + ForkedTree::from_leaves_and_path_unchecked( + parameters, + leaves + .iter() + .filter(|leaf| C::tree_index(leaf).into() == tree_index) + .map(|leaf| parameters.digest(leaf)) + .collect(), + path, + ) + }, + ))) + } +} + impl AsRef<[T; N]> for TreeArray where C: Configuration + ?Sized, diff --git a/manta-crypto/src/merkle_tree/fork.rs b/manta-crypto/src/merkle_tree/fork.rs index 8e6865e2e..b4a44b957 100644 --- a/manta-crypto/src/merkle_tree/fork.rs +++ b/manta-crypto/src/merkle_tree/fork.rs @@ -930,6 +930,27 @@ where } } +impl ForkedTree> +where + C: Configuration + ?Sized, + LeafDigest: Clone + Default, + InnerDigest: Clone + Default + PartialEq, +{ + /// Builds a new [`ForkedTree`] from `leaf_digests` and `path` without checking that + /// `path` is consistent with the leaves and that it is a [`CurrentPath`]. + #[inline] + pub fn from_leaves_and_path_unchecked( + parameters: &Parameters, + leaf_digests: Vec>, + path: Path, + ) -> Self { + Self::new( + Partial::from_leaves_and_path_unchecked(parameters, leaf_digests, path), + parameters, + ) + } +} + impl Tree for ForkedTree where C: Configuration + ?Sized, diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index f9cc34aae..cdf276af2 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -21,6 +21,7 @@ use crate::merkle_tree::{ capacity, inner_tree::{BTreeMap, InnerMap, PartialInnerTree}, + node::Parity, Configuration, CurrentPath, InnerDigest, Leaf, LeafDigest, MerkleTree, Node, Parameters, Path, PathError, Root, Tree, WithProofs, }; @@ -85,6 +86,37 @@ where } } + /// Builds a new [`Partial`] from `leaf_digests` and `path` without checking that + /// `path` is consistent with the leaves and that it is a [`CurrentPath`]. + #[inline] + pub fn from_leaves_and_path_unchecked( + parameters: &Parameters, + leaf_digests: Vec>, + path: Path, + ) -> Self + where + M: Default, + InnerDigest: Default + PartialEq, + { + let n = leaf_digests.len(); + if n == 0 { + Self::new_unchecked(leaf_digests, Default::default()) + } else { + let base = match Parity::from_index(n - 1) { + Parity::Left => parameters.join_leaves(&leaf_digests[n - 1], &path.sibling_digest), + Parity::Right => parameters.join_leaves(&path.sibling_digest, &leaf_digests[n - 1]), + }; + Self::new_unchecked( + leaf_digests, + PartialInnerTree::from_current( + parameters, + base, + CurrentPath::from_path_unchecked(path).inner_path, + ), + ) + } + } + /// Returns the leaf digests currently stored in the merkle tree. /// /// # Note From 6496e486481942667ec9b72e20f03b11abaf8f11 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 17:58:47 +0100 Subject: [PATCH 05/58] accumulator trait --- manta-crypto/src/accumulator.rs | 12 ++++++ manta-crypto/src/merkle_tree/forest.rs | 51 +++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 0b09f235c..979fafc87 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -17,10 +17,12 @@ //! Dynamic Cryptographic Accumulators use crate::eclair::alloc::{mode::Derived, Allocate, Allocator, Constant, Variable}; +use alloc::vec::Vec; use core::{fmt::Debug, hash::Hash}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; +use manta_util::BoxArray; /// Accumulator Membership Model Types pub trait Types { @@ -202,6 +204,16 @@ pub trait OptimizedAccumulator: Accumulator { } } +/// From Proofs and Witnesses +pub trait FromItemsAndWitnesses: Accumulator { + /// Builds a new [`Self`] from `items` and `proofs`. + fn from_items_and_witnesses( + model: &Self::Model, + items: Vec, + witnesses: BoxArray, + ) -> Self; +} + /// Accumulator Membership Proof #[cfg_attr( feature = "serde", diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 2f2a6a166..7110680a8 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -24,8 +24,8 @@ use crate::{ accumulator::{ - self, Accumulator, ConstantCapacityAccumulator, ExactSizeAccumulator, MembershipProof, - OptimizedAccumulator, + self, Accumulator, ConstantCapacityAccumulator, ExactSizeAccumulator, + FromItemsAndWitnesses, MembershipProof, OptimizedAccumulator, }, merkle_tree::{ fork::ForkedTree, @@ -590,6 +590,53 @@ where } } +impl FromItemsAndWitnesses for TreeArrayMerkleForest, N> +where + C: Configuration + ?Sized, + C::Index: FixedIndex, + Parameters: Clone, + LeafDigest: Clone + Default + PartialEq, + InnerDigest: Clone + Default + PartialEq, +{ + #[inline] + fn from_items_and_witnesses( + model: &Self::Model, + items: Vec, + witnesses: BoxArray, + ) -> Self { + Self::from_forest( + TreeArray::, N>::from_leaves_and_current_paths_unchecked( + model, items, witnesses, + ), + model.clone(), + ) + } +} + +impl FromItemsAndWitnesses + for TreeArrayMerkleForest>, N> +where + C: Configuration + ?Sized, + C::Index: FixedIndex, + Parameters: Clone, + LeafDigest: Clone + Default + PartialEq, + InnerDigest: Clone + Default + PartialEq, +{ + #[inline] + fn from_items_and_witnesses( + model: &Self::Model, + items: Vec, + witnesses: BoxArray, + ) -> Self { + Self::from_forest( + TreeArray::>, N>::from_leaves_and_current_paths_unchecked( + model, items, witnesses, + ), + model.clone(), + ) + } +} + impl AsRef<[T; N]> for TreeArray where C: Configuration + ?Sized, From e6d73f96a800d98ef664cd0b6a7a44a44c15ba92 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 18:24:13 +0100 Subject: [PATCH 06/58] signer functions --- .../src/wallet/signer/functions.rs | 28 ++++++++++++------- manta-accounting/src/wallet/signer/mod.rs | 25 +++++++++-------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 8d30ebc93..b30c49fba 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -32,7 +32,7 @@ use crate::{ Address, Asset, AssociatedData, Authorization, AuthorizationContext, FullParametersRef, IdentifiedAsset, Identifier, IdentityProof, Note, Nullifier, Parameters, PreSender, ProvingContext, Receiver, Sender, Shape, SpendingKey, Transfer, TransferPost, Utxo, - UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoMembershipProof, + UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoAccumulatorWitness, }, wallet::signer::{ AccountTable, BalanceUpdate, Checkpoint, Configuration, InitialSyncData, SignError, @@ -42,12 +42,12 @@ use crate::{ }; use alloc::{vec, vec::Vec}; use manta_crypto::{ - accumulator::{Accumulator, ItemHashFunction, OptimizedAccumulator}, + accumulator::{Accumulator, FromItemsAndWitnesses, ItemHashFunction, OptimizedAccumulator}, rand::Rand, }; use manta_util::{ array_map, cmp::Independence, into_array_unchecked, iter::IteratorExt, persistence::Rollback, - vec::VecExt, Array, + vec::VecExt, BoxArray, }; /// Returns the default account for `accounts`. @@ -938,15 +938,15 @@ where #[inline] pub fn intial_sync( parameters: &SignerParameters, - authorization_context: &mut AuthorizationContext, assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator: &mut C::UtxoAccumulator, + utxo_accumulator_model: &UtxoAccumulatorModel, request: InitialSyncData, - rng: &mut C::Rng, ) -> Result, SyncError> where C: Configuration, + C::UtxoAccumulator: FromItemsAndWitnesses, { let InitialSyncData { utxo_data, @@ -954,15 +954,14 @@ where nullifier_data, } = request; let response = initial_sync_with::( - authorization_context, assets, checkpoint, utxo_accumulator, + utxo_accumulator_model, ¶meters.parameters, utxo_data, membership_proof_data, nullifier_data, - rng, ); utxo_accumulator.commit(); Ok(response) @@ -972,19 +971,28 @@ where #[allow(clippy::too_many_arguments)] #[inline] fn initial_sync_with( - authorization_context: &mut AuthorizationContext, assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator: &mut C::UtxoAccumulator, + utxo_accumulator_model: &UtxoAccumulatorModel, parameters: &Parameters, utxos: Vec>, - membership_proof_data: Array, NUMBER_OF_PROOFS>, + membership_proof_data: BoxArray, NUMBER_OF_PROOFS>, nullifiers: Vec>, - rng: &mut C::Rng, ) -> SyncResponse where C: Configuration, + C::UtxoAccumulator: FromItemsAndWitnesses, { + let accumulator = C::UtxoAccumulator::from_items_and_witnesses( + utxo_accumulator_model, + utxos + .iter() + .map(|utxo| item_hash::(parameters, utxo)) + .collect(), + membership_proof_data, + ); + *utxo_accumulator = accumulator; checkpoint.update_from_nullifiers(nullifiers.len()); checkpoint.update_from_utxo_accumulator(utxo_accumulator); SyncResponse { diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 4fe36855f..6a661c602 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -33,7 +33,7 @@ use crate::{ canonical::{MultiProvingContext, Transaction, TransactionData}, Address, Asset, AuthorizationContext, IdentifiedAsset, Identifier, IdentityProof, Note, Nullifier, Parameters, ProofSystemError, SpendingKey, TransferPost, Utxo, - UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoMembershipProof, + UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoAccumulatorWitness, UtxoMembershipProof, }, wallet::ledger::{self, Data}, }; @@ -116,12 +116,12 @@ where bound( deserialize = r" Utxo: Deserialize<'de>, - UtxoMembershipProof: Deserialize<'de>, + UtxoAccumulatorWitness: Deserialize<'de>, Nullifier: Deserialize<'de>, ", serialize = r" Utxo: Serialize, - UtxoMembershipProof: Serialize, + UtxoAccumulatorWitness: Serialize, Nullifier: Serialize, ", ), @@ -131,12 +131,12 @@ where )] #[derive(derivative::Derivative)] #[derivative( - Clone(bound = "Utxo: Clone, UtxoMembershipProof: Clone, Nullifier: Clone"), - Debug(bound = "Utxo: Debug, UtxoMembershipProof: Debug, Nullifier: Debug"), - Eq(bound = "Utxo: Eq, UtxoMembershipProof: Eq, Nullifier: Eq"), - Hash(bound = "Utxo: Hash, UtxoMembershipProof: Hash, Nullifier: Hash"), + Clone(bound = "Utxo: Clone, UtxoAccumulatorWitness: Clone, Nullifier: Clone"), + Debug(bound = "Utxo: Debug, UtxoAccumulatorWitness: Debug, Nullifier: Debug"), + Eq(bound = "Utxo: Eq, UtxoAccumulatorWitness: Eq, Nullifier: Eq"), + Hash(bound = "Utxo: Hash, UtxoAccumulatorWitness: Hash, Nullifier: Hash"), PartialEq( - bound = "Utxo: PartialEq, UtxoMembershipProof: PartialEq, Nullifier: PartialEq" + bound = "Utxo: PartialEq, UtxoAccumulatorWitness: PartialEq, Nullifier: PartialEq" ) )] pub struct InitialSyncData @@ -150,7 +150,7 @@ where pub nullifier_data: Vec>, /// Membership Proof Data - pub membership_proof_data: manta_util::Array, NUMBER_OF_PROOFS>, + pub membership_proof_data: manta_util::BoxArray, NUMBER_OF_PROOFS>, } /// Signer Synchronization Data @@ -708,8 +708,11 @@ pub trait Configuration: transfer::Configuration { + DeriveAddresses; /// [`Utxo`] Accumulator Type - type UtxoAccumulator: Accumulator, Model = UtxoAccumulatorModel> - + ExactSizeAccumulator + type UtxoAccumulator: Accumulator< + Item = UtxoAccumulatorItem, + Model = UtxoAccumulatorModel, + Witness = UtxoAccumulatorWitness, + > + ExactSizeAccumulator + OptimizedAccumulator + Rollback; From 8444d0127830324e0d24d4da549eec3587e3eded Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 18:47:01 +0100 Subject: [PATCH 07/58] signer initial sync --- .../src/wallet/signer/functions.rs | 26 +++++++++---------- manta-accounting/src/wallet/signer/mod.rs | 23 +++++++++++++++- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index b30c49fba..adb762cfc 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -941,7 +941,6 @@ pub fn intial_sync( assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator: &mut C::UtxoAccumulator, - utxo_accumulator_model: &UtxoAccumulatorModel, request: InitialSyncData, ) -> Result, SyncError> where @@ -953,16 +952,16 @@ where membership_proof_data, nullifier_data, } = request; - let response = initial_sync_with::( + let (accumulator, response) = initial_sync_with::( assets, checkpoint, - utxo_accumulator, - utxo_accumulator_model, + utxo_accumulator.model(), ¶meters.parameters, utxo_data, membership_proof_data, nullifier_data, ); + *utxo_accumulator = accumulator; utxo_accumulator.commit(); Ok(response) } @@ -973,13 +972,12 @@ where fn initial_sync_with( assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, - utxo_accumulator: &mut C::UtxoAccumulator, utxo_accumulator_model: &UtxoAccumulatorModel, parameters: &Parameters, utxos: Vec>, membership_proof_data: BoxArray, NUMBER_OF_PROOFS>, nullifiers: Vec>, -) -> SyncResponse +) -> (C::UtxoAccumulator, SyncResponse) where C: Configuration, C::UtxoAccumulator: FromItemsAndWitnesses, @@ -992,13 +990,15 @@ where .collect(), membership_proof_data, ); - *utxo_accumulator = accumulator; checkpoint.update_from_nullifiers(nullifiers.len()); - checkpoint.update_from_utxo_accumulator(utxo_accumulator); - SyncResponse { - checkpoint: checkpoint.clone(), - balance_update: BalanceUpdate::Full { - assets: assets.assets().into(), + checkpoint.update_from_utxo_accumulator(&accumulator); + ( + accumulator, + SyncResponse { + checkpoint: checkpoint.clone(), + balance_update: BalanceUpdate::Full { + assets: assets.assets().into(), + }, }, - } + ) } diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 6a661c602..fe79bc683 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -40,7 +40,10 @@ use crate::{ use alloc::{boxed::Box, vec::Vec}; use core::{convert::Infallible, fmt::Debug, hash::Hash}; use manta_crypto::{ - accumulator::{Accumulator, ExactSizeAccumulator, ItemHashFunction, OptimizedAccumulator}, + accumulator::{ + Accumulator, ExactSizeAccumulator, FromItemsAndWitnesses, ItemHashFunction, + OptimizedAccumulator, + }, rand::{CryptoRng, FromEntropy, RngCore}, }; use manta_util::{future::LocalBoxFutureResult, persistence::Rollback}; @@ -1250,6 +1253,24 @@ where } false } + + /// + #[inline] + pub fn initial_sync( + &mut self, + request: InitialSyncData, + ) -> Result, SyncError> + where + C::UtxoAccumulator: FromItemsAndWitnesses, + { + functions::intial_sync( + &self.parameters, + &mut self.state.assets, + &mut self.state.checkpoint, + &mut self.state.utxo_accumulator, + request, + ) + } } impl Connection for Signer From 1c18f589e3ab94316f541499c966ef749b934e8a Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 15 Mar 2023 19:19:36 +0100 Subject: [PATCH 08/58] wallet has method as well --- manta-accounting/src/wallet/mod.rs | 69 ++++++++++++++++++- .../src/wallet/signer/functions.rs | 14 ++-- manta-accounting/src/wallet/signer/mod.rs | 52 ++++++++------ manta-crypto/src/accumulator.rs | 10 +-- manta-crypto/src/merkle_tree/forest.rs | 18 +++-- manta-pay/src/signer/client/http.rs | 14 +++- manta-pay/src/signer/client/websocket.rs | 14 +++- manta-pay/src/signer/mod.rs | 3 + 8 files changed, 147 insertions(+), 47 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index f880b34f8..8b4138ed4 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -37,9 +37,9 @@ use crate::{ balance::{BTreeMapBalanceState, BalanceState}, ledger::ReadResponse, signer::{ - BalanceUpdate, IdentityRequest, IdentityResponse, SignError, SignRequest, SignResponse, - SignWithTransactionDataResponse, SyncData, SyncError, SyncRequest, SyncResponse, - TransactionDataRequest, TransactionDataResponse, + BalanceUpdate, IdentityRequest, IdentityResponse, InitialSyncData, SignError, + SignRequest, SignResponse, SignWithTransactionDataResponse, SyncData, SyncError, + SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, }, }, }; @@ -265,6 +265,24 @@ where Ok(()) } + /// + #[inline] + pub async fn initial_sync(&mut self) -> Result<(), Error> + where + L: ledger::Read, Checkpoint = S::Checkpoint>, + { + let ReadResponse { + should_continue: _, + data, + } = self + .ledger + .read(&self.checkpoint) + .await + .map_err(Error::LedgerConnectionError)?; + self.signer_initial_sync(data).await?; + Ok(()) + } + /// Pulls data from the ledger, synchronizing the wallet and balance state. This method returns /// a [`ControlFlow`] for matching against to determine if the wallet requires more /// synchronization. @@ -350,6 +368,51 @@ where } } + /// + #[inline] + async fn signer_initial_sync( + &mut self, + request: InitialSyncData, + ) -> Result<(), Error> { + match self + .signer + .initial_sync(request) + .await + .map_err(Error::SignerConnectionError)? + { + Ok(SyncResponse { + checkpoint, + balance_update, + }) => { + match balance_update { + BalanceUpdate::Partial { deposit, withdraw } => { + self.assets.deposit_all(deposit); + if !self.assets.withdraw_all(withdraw) { + return Err(Error::Inconsistency(InconsistencyError::WalletBalance)); + } + } + BalanceUpdate::Full { assets } => { + self.assets.clear(); + self.assets.deposit_all(assets); + } + } + self.checkpoint = checkpoint; + Ok(()) + } + Err(SyncError::InconsistentSynchronization { checkpoint }) => { + if checkpoint < self.checkpoint { + self.checkpoint = checkpoint; + } + Err(Error::Inconsistency( + InconsistencyError::SignerSynchronization, + )) + } + Err(SyncError::MissingProofAuthorizationKey) => { + Err(Error::MissingProofAuthorizationKey) + } + } + } + /// Checks if `transaction` can be executed on the balance state of `self`, returning the /// kind of update that should be performed on the balance state if the transaction is /// successfully posted to the ledger. diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index adb762cfc..3f5d3663b 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -47,7 +47,7 @@ use manta_crypto::{ }; use manta_util::{ array_map, cmp::Independence, into_array_unchecked, iter::IteratorExt, persistence::Rollback, - vec::VecExt, BoxArray, + vec::VecExt, }; /// Returns the default account for `accounts`. @@ -936,23 +936,22 @@ where /// Updates `assets`, `checkpoint` and `utxo_accumulator`, returning the new asset distribution. #[inline] -pub fn intial_sync( +pub fn intial_sync( parameters: &SignerParameters, assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator: &mut C::UtxoAccumulator, - request: InitialSyncData, + request: InitialSyncData, ) -> Result, SyncError> where C: Configuration, - C::UtxoAccumulator: FromItemsAndWitnesses, { let InitialSyncData { utxo_data, membership_proof_data, nullifier_data, } = request; - let (accumulator, response) = initial_sync_with::( + let (accumulator, response) = initial_sync_with::( assets, checkpoint, utxo_accumulator.model(), @@ -969,18 +968,17 @@ where /// Updates the internal ledger state, returning the new asset distribution. #[allow(clippy::too_many_arguments)] #[inline] -fn initial_sync_with( +fn initial_sync_with( assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator_model: &UtxoAccumulatorModel, parameters: &Parameters, utxos: Vec>, - membership_proof_data: BoxArray, NUMBER_OF_PROOFS>, + membership_proof_data: Vec>, nullifiers: Vec>, ) -> (C::UtxoAccumulator, SyncResponse) where C: Configuration, - C::UtxoAccumulator: FromItemsAndWitnesses, { let accumulator = C::UtxoAccumulator::from_items_and_witnesses( utxo_accumulator_model, diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index fe79bc683..fc15209bb 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -79,6 +79,12 @@ where request: SyncRequest, ) -> LocalBoxFutureResult, Self::Error>; + /// + fn initial_sync( + &mut self, + request: InitialSyncData, + ) -> LocalBoxFutureResult, Self::Error>; + /// Signs a transaction and returns the ledger transfer posts if successful. fn sign( &mut self, @@ -142,7 +148,7 @@ where bound = "Utxo: PartialEq, UtxoAccumulatorWitness: PartialEq, Nullifier: PartialEq" ) )] -pub struct InitialSyncData +pub struct InitialSyncData where C: transfer::Configuration + ?Sized, { @@ -153,7 +159,7 @@ where pub nullifier_data: Vec>, /// Membership Proof Data - pub membership_proof_data: manta_util::BoxArray, NUMBER_OF_PROOFS>, + pub membership_proof_data: Vec>, } /// Signer Synchronization Data @@ -716,6 +722,7 @@ pub trait Configuration: transfer::Configuration { Model = UtxoAccumulatorModel, Witness = UtxoAccumulatorWitness, > + ExactSizeAccumulator + + FromItemsAndWitnesses + OptimizedAccumulator + Rollback; @@ -1126,6 +1133,21 @@ where ) } + /// + #[inline] + pub fn initial_sync( + &mut self, + request: InitialSyncData, + ) -> Result, SyncError> { + functions::intial_sync( + &self.parameters, + &mut self.state.assets, + &mut self.state.checkpoint, + &mut self.state.utxo_accumulator, + request, + ) + } + /// Generates an [`IdentityProof`] for `identified_asset` by /// signing a virtual [`ToPublic`](transfer::canonical::ToPublic) transaction. #[inline] @@ -1253,24 +1275,6 @@ where } false } - - /// - #[inline] - pub fn initial_sync( - &mut self, - request: InitialSyncData, - ) -> Result, SyncError> - where - C::UtxoAccumulator: FromItemsAndWitnesses, - { - functions::intial_sync( - &self.parameters, - &mut self.state.assets, - &mut self.state.checkpoint, - &mut self.state.utxo_accumulator, - request, - ) - } } impl Connection for Signer @@ -1289,6 +1293,14 @@ where Box::pin(async move { Ok(self.sync(request)) }) } + #[inline] + fn initial_sync( + &mut self, + request: InitialSyncData, + ) -> LocalBoxFutureResult, Self::Error> { + Box::pin(async move { Ok(self.initial_sync(request)) }) + } + #[inline] fn sign( &mut self, diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 979fafc87..d08348392 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -22,7 +22,6 @@ use core::{fmt::Debug, hash::Hash}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; -use manta_util::BoxArray; /// Accumulator Membership Model Types pub trait Types { @@ -204,13 +203,16 @@ pub trait OptimizedAccumulator: Accumulator { } } -/// From Proofs and Witnesses -pub trait FromItemsAndWitnesses: Accumulator { +/// From Items and Witnesses +pub trait FromItemsAndWitnesses: Accumulator { + /// Number of Proofs + const NUMBER_OF_PROOFS: usize; + /// Builds a new [`Self`] from `items` and `proofs`. fn from_items_and_witnesses( model: &Self::Model, items: Vec, - witnesses: BoxArray, + witnesses: Vec, ) -> Self; } diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 7110680a8..7cc09c4ff 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -540,7 +540,7 @@ where pub fn from_leaves_and_current_paths_unchecked( parameters: &Parameters, leaves: Vec>, - paths: BoxArray, N>, + paths: Vec>, ) -> Self { TreeArray::new(BoxArray::from_iter(paths.into_iter().enumerate().map( |(tree_index, path)| { @@ -572,7 +572,7 @@ where pub fn from_leaves_and_current_paths_unchecked( parameters: &Parameters, leaves: Vec>, - paths: BoxArray, N>, + paths: Vec>, ) -> Self { TreeArray::new(BoxArray::from_iter(paths.into_iter().enumerate().map( |(tree_index, path)| { @@ -590,7 +590,7 @@ where } } -impl FromItemsAndWitnesses for TreeArrayMerkleForest, N> +impl FromItemsAndWitnesses for TreeArrayMerkleForest, N> where C: Configuration + ?Sized, C::Index: FixedIndex, @@ -598,12 +598,15 @@ where LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { + const NUMBER_OF_PROOFS: usize = N; + #[inline] fn from_items_and_witnesses( model: &Self::Model, items: Vec, - witnesses: BoxArray, + witnesses: Vec, ) -> Self { + assert_eq!(witnesses.len(), N); Self::from_forest( TreeArray::, N>::from_leaves_and_current_paths_unchecked( model, items, witnesses, @@ -613,7 +616,7 @@ where } } -impl FromItemsAndWitnesses +impl FromItemsAndWitnesses for TreeArrayMerkleForest>, N> where C: Configuration + ?Sized, @@ -622,12 +625,15 @@ where LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { + const NUMBER_OF_PROOFS: usize = N; + #[inline] fn from_items_and_witnesses( model: &Self::Model, items: Vec, - witnesses: BoxArray, + witnesses: Vec, ) -> Self { + assert_eq!(witnesses.len(), N); Self::from_forest( TreeArray::>, N>::from_leaves_and_current_paths_unchecked( model, items, witnesses, diff --git a/manta-pay/src/signer/client/http.rs b/manta-pay/src/signer/client/http.rs index dc3690228..20448ed48 100644 --- a/manta-pay/src/signer/client/http.rs +++ b/manta-pay/src/signer/client/http.rs @@ -20,9 +20,9 @@ use crate::{ config::{utxo::Address, Config}, signer::{ client::network::{Message, Network}, - AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, SignError, - SignRequest, SignResponse, SignWithTransactionDataResult, SyncError, SyncRequest, - SyncResponse, TransactionDataRequest, TransactionDataResponse, + AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, InitialSyncData, + SignError, SignRequest, SignResponse, SignWithTransactionDataResult, SyncError, + SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, }, }; use alloc::boxed::Box; @@ -102,6 +102,14 @@ impl signer::Connection for Client { Box::pin(self.post_request("sync", request)) } + #[inline] + fn initial_sync( + &mut self, + request: InitialSyncData, + ) -> LocalBoxFutureResult, Self::Error> { + Box::pin(self.post_request("initial_sync", request)) + } + #[inline] fn sign( &mut self, diff --git a/manta-pay/src/signer/client/websocket.rs b/manta-pay/src/signer/client/websocket.rs index 2ff6e70be..376869a5f 100644 --- a/manta-pay/src/signer/client/websocket.rs +++ b/manta-pay/src/signer/client/websocket.rs @@ -21,9 +21,9 @@ use crate::{ config::{utxo::Address, Config}, signer::{ - AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, SignError, - SignRequest, SignResponse, SignWithTransactionDataResult, SyncError, SyncRequest, - SyncResponse, TransactionDataRequest, TransactionDataResponse, + AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, InitialSyncData, + SignError, SignRequest, SignResponse, SignWithTransactionDataResult, SyncError, + SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, }, }; use alloc::boxed::Box; @@ -140,6 +140,14 @@ impl signer::Connection for Client { Box::pin(self.send("sync", request)) } + #[inline] + fn initial_sync( + &mut self, + request: InitialSyncData, + ) -> LocalBoxFutureResult, Self::Error> { + Box::pin(self.send("initial_sync", request)) + } + #[inline] fn sign( &mut self, diff --git a/manta-pay/src/signer/mod.rs b/manta-pay/src/signer/mod.rs index 819a2ab34..71808ebde 100644 --- a/manta-pay/src/signer/mod.rs +++ b/manta-pay/src/signer/mod.rs @@ -40,6 +40,9 @@ pub mod functions; /// Synchronization Request pub type SyncRequest = signer::SyncRequest; +/// Initial Synchronization Data +pub type InitialSyncData = signer::InitialSyncData; + /// Synchronization Response pub type SyncResponse = signer::SyncResponse; From 091eb31034662e2764017638c6fe40c74e1ee436 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Thu, 16 Mar 2023 19:32:24 +0100 Subject: [PATCH 09/58] tests --- manta-accounting/src/wallet/test/mod.rs | 17 +- manta-crypto/src/merkle_tree/forest.rs | 10 +- manta-crypto/src/merkle_tree/inner_tree.rs | 8 +- manta-crypto/src/merkle_tree/partial.rs | 8 +- manta-crypto/src/merkle_tree/test.rs | 190 ++++++++++++++++++++- manta-pay/src/bin/simulation.rs | 2 + manta-pay/src/simulation/ledger/mod.rs | 68 +++++++- manta-pay/src/simulation/mod.rs | 28 ++- 8 files changed, 306 insertions(+), 25 deletions(-) diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index 7b29cd6e5..bf92913c4 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -24,7 +24,7 @@ use crate::{ transfer::{canonical::Transaction, Address, Asset, Configuration, TransferPost}, wallet::{ ledger, - signer::{self, SyncData}, + signer::{self, InitialSyncData, SyncData}, BalanceState, Error, Wallet, }, }; @@ -993,8 +993,14 @@ impl Config { C: Configuration, C::AssetValue: AddAssign + SampleUniform, for<'v> &'v C::AssetValue: CheckedSub, - L: Ledger + PublicBalanceOracle, - S: signer::Connection, + L: Ledger + + PublicBalanceOracle + + ledger::Read< + InitialSyncData, + Checkpoint = >>::Checkpoint, + >, + Error: Debug, + S: signer::Connection>>::Checkpoint>, S::Error: Debug, B: BalanceState, R: CryptoRng + RngCore, @@ -1027,6 +1033,11 @@ impl Config { .expect("Wallet should have address") .expect("Missing spending key"); simulation.addresses.lock().insert(address); + actor + .wallet + .initial_sync() + .await + .expect("Error during initial sync"); } let mut simulator = sim::Simulator::new(sim::ActionSim(simulation), actors); let initial_balances = diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 7cc09c4ff..1927ea48a 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -537,7 +537,7 @@ where /// the `paths` are consistent with the leaves and that they are /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. #[inline] - pub fn from_leaves_and_current_paths_unchecked( + pub fn from_leaves_and_paths_unchecked( parameters: &Parameters, leaves: Vec>, paths: Vec>, @@ -569,7 +569,7 @@ where /// the `paths` are consistent with the leaves and that they are /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. #[inline] - pub fn from_leaves_and_current_paths_unchecked( + pub fn from_leaves_and_paths_unchecked( parameters: &Parameters, leaves: Vec>, paths: Vec>, @@ -608,9 +608,7 @@ where ) -> Self { assert_eq!(witnesses.len(), N); Self::from_forest( - TreeArray::, N>::from_leaves_and_current_paths_unchecked( - model, items, witnesses, - ), + TreeArray::, N>::from_leaves_and_paths_unchecked(model, items, witnesses), model.clone(), ) } @@ -635,7 +633,7 @@ where ) -> Self { assert_eq!(witnesses.len(), N); Self::from_forest( - TreeArray::>, N>::from_leaves_and_current_paths_unchecked( + TreeArray::>, N>::from_leaves_and_paths_unchecked( model, items, witnesses, ), model.clone(), diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index 35b7004d9..7ab7dddd0 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -25,7 +25,7 @@ use crate::merkle_tree::{ path_length, Configuration, InnerDigest, Node, Parameters, Parity, }; use alloc::collections::btree_map; -use core::{fmt::Debug, hash::Hash, iter::FusedIterator, marker::PhantomData, ops::Index}; +use core::{default, fmt::Debug, hash::Hash, iter::FusedIterator, marker::PhantomData, ops::Index}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; @@ -774,6 +774,12 @@ where { self.inner_tree.path(leaf_index) } + + /// + #[inline] + pub fn reset_starting_leaf_index(&mut self, default: Node) { + self.starting_leaf_index = default; + } } impl PartialInnerTree> diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index cdf276af2..806bb7ed2 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -106,14 +106,18 @@ where Parity::Left => parameters.join_leaves(&leaf_digests[n - 1], &path.sibling_digest), Parity::Right => parameters.join_leaves(&path.sibling_digest, &leaf_digests[n - 1]), }; - Self::new_unchecked( + let mut partial_tree = Self::new_unchecked( leaf_digests, PartialInnerTree::from_current( parameters, base, CurrentPath::from_path_unchecked(path).inner_path, ), - ) + ); + partial_tree + .inner_digests + .reset_starting_leaf_index(Default::default()); + partial_tree } } diff --git a/manta-crypto/src/merkle_tree/test.rs b/manta-crypto/src/merkle_tree/test.rs index 0ef3a42e7..f80e9ec27 100644 --- a/manta-crypto/src/merkle_tree/test.rs +++ b/manta-crypto/src/merkle_tree/test.rs @@ -17,16 +17,23 @@ //! Testing Framework use crate::{ + accumulator::{Accumulator, FromItemsAndWitnesses}, merkle_tree::{ + forest::{self, Forest, MerkleForest, TreeArrayMerkleForest}, + fork::ForkedTree, + full::{Full, FullMerkleTree}, + partial::{Partial, PartialMerkleTree}, Configuration, HashConfiguration, IdentityLeafHash, InnerDigest, InnerHash, InnerHashParameters, Leaf, LeafHashParameters, MerkleTree, Parameters, Path, Tree, WithProofs, }, - rand::{RngCore, Sample}, + rand::{OsRng, Rand, RngCore, Sample}, }; use alloc::string::String; use core::{fmt::Debug, hash::Hash, marker::PhantomData}; +use super::forest::FixedIndex; + /// Hash Parameter Sampling pub trait HashParameterSampling: HashConfiguration { /// Leaf Hash Parameter Distribution @@ -279,3 +286,184 @@ where let _ = (distribution, rng); } } + +/// +#[test] +fn test_from_leaves_and_path() { + let mut rng = OsRng; + const HEIGHT: usize = 7; + type Config = Test; + let parameters = Parameters::::sample(Default::default(), &mut rng); + let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); + let inner_element_index = rng.gen_range(0..number_of_insertions - 3); // make sure this one isn't the last nor its sibling + println!("{number_of_insertions}, {inner_element_index}"); + let mut tree = FullMerkleTree::::new(parameters); + let mut insertions = Vec::::with_capacity(number_of_insertions); + for _ in 0..number_of_insertions { + insertions.push(rng.gen()); + } + for leaf in &insertions { + tree.insert(leaf); + } + let forked_tree = ForkedTree::>::new(tree.tree.clone(), ¶meters); + let path = tree.current_path(); + let partial_tree = PartialMerkleTree::> { + parameters, + tree: Partial::from_leaves_and_path_unchecked( + ¶meters, + insertions.clone(), + path.clone().into(), + ), + }; + let forked_partial_tree = ForkedTree::>::from_leaves_and_path_unchecked( + ¶meters, + insertions.clone(), + path.clone().into(), + ); + let root = tree.root().clone(); + let partial_root = partial_tree.root().clone(); + let forked_root = forked_tree.root().clone(); + let forked_partial_root = forked_partial_tree.root().clone(); + assert_eq!(root, partial_root, "Roots must be equal"); + assert_eq!(root, forked_root, "Roots must be equal"); + assert_eq!(root, forked_partial_root, "Roots must be equal"); + let proof_full_inner = tree + .prove(&insertions[inner_element_index]) + .expect("Failed to generate proof"); + let proof_partial_inner = partial_tree + .prove(&insertions[inner_element_index]) + .expect("Failed to generate proof"); + assert!( + proof_full_inner.verify(¶meters, &insertions[inner_element_index], &mut ()), + "Inner proof in the full tree must be valid" + ); + assert!( + !proof_partial_inner.verify(¶meters, &insertions[inner_element_index], &mut ()), + "Inner proof in the partial tree must be invalid" + ); + let proof_full = tree + .prove(&insertions[number_of_insertions - 1]) + .expect("Failed to generate proof"); + let proof_partial = partial_tree + .prove(&insertions[number_of_insertions - 1]) + .expect("Failed to generate proof"); + assert!( + proof_full.verify(¶meters, &insertions[number_of_insertions - 1], &mut ()), + "Final proof in the full tree must be valid" + ); + assert!( + proof_partial.verify(¶meters, &insertions[number_of_insertions - 1], &mut ()), + "Final proof in the partial tree must be valid" + ); +} + +/// +#[derive(PartialEq)] +pub enum Index { + /// + Zero, + + /// + One, +} + +impl From for usize { + fn from(value: Index) -> Self { + match value { + Index::Zero => 0, + Index::One => 1, + } + } +} + +impl FixedIndex<2> for Index { + fn from_index(index: usize) -> Self { + if index % 2 == 0 { + Index::Zero + } else { + Index::One + } + } +} + +impl forest::Configuration for Test { + type Index = Index; + fn tree_index(leaf: &Leaf) -> Self::Index { + let parity = leaf % 2; + if parity == 0 { + Index::Zero + } else { + Index::One + } + } +} +/// +#[test] +fn test_from_leaves_and_path_forest() { + let mut rng = OsRng; + const HEIGHT: usize = 7; + type Config = Test; + let parameters = Parameters::::sample(Default::default(), &mut rng); + let mut forest = + TreeArrayMerkleForest::>, 2>::new(parameters); + let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); + let mut insertions = Vec::::with_capacity(number_of_insertions); + for _ in 0..number_of_insertions { + insertions.push(rng.gen()); + } + for leaf in &insertions { + forest.insert(leaf); + } + let path_1 = Path::from(forest.forest.get(Index::Zero).current_path()); + let path_2 = Path::from(forest.forest.get(Index::One).current_path()); + let paths = vec![path_1, path_2]; + let partial_forest = + TreeArrayMerkleForest::<_, ForkedTree<_, Partial>, 2>::from_items_and_witnesses( + ¶meters, + insertions.clone(), + paths, + ); + for leaf in &insertions { + assert_eq!(forest.output_from(leaf), partial_forest.output_from(leaf)); + } +} + +/// +#[test] +fn visual_test_with_strings() { + const HEIGHT: usize = 5; + let mut rng = OsRng; + let parameters = Parameters::>::sample(Default::default(), &mut rng); + let mut tree = FullMerkleTree::>::new(parameters); + let insertions = vec![ + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", + ] + .into_iter() + .map(String::from) + .collect::>(); + for leaf in &insertions { + tree.insert(leaf); + } + let root = tree.root().clone(); + let path = tree.current_path(); + println!("{path:?}"); + let partial_tree = PartialMerkleTree::> { + parameters, + tree: Partial::from_leaves_and_path_unchecked(¶meters, insertions.clone(), path.into()), + }; + let second_root = partial_tree.root().clone(); + assert_eq!(root, second_root); + const INNER_ELEMENT_INDEX: usize = 10; // k + let proof = tree.prove(&insertions[INNER_ELEMENT_INDEX]).unwrap(); + println!("{proof:?}"); + let proof_2 = partial_tree + .prove(&insertions[INNER_ELEMENT_INDEX]) + .unwrap(); + println!("{proof_2:?}"); + let proof_3 = tree.prove(&insertions[insertions.len() - 1]).unwrap(); + let proof_4 = partial_tree + .prove(&insertions[insertions.len() - 1]) + .unwrap(); + println!("{proof_3:?}"); + println!("{proof_4:?}"); +} diff --git a/manta-pay/src/bin/simulation.rs b/manta-pay/src/bin/simulation.rs index 1d43790a9..afdb3f916 100644 --- a/manta-pay/src/bin/simulation.rs +++ b/manta-pay/src/bin/simulation.rs @@ -46,3 +46,5 @@ pub fn main() { .exit(), } } + +// cargo run --release --package manta-pay --all-features --bin simulation 3 10 2 1000 diff --git a/manta-pay/src/simulation/ledger/mod.rs b/manta-pay/src/simulation/ledger/mod.rs index e109d4cb5..d84395c80 100644 --- a/manta-pay/src/simulation/ledger/mod.rs +++ b/manta-pay/src/simulation/ledger/mod.rs @@ -19,12 +19,15 @@ // TODO: How to model existential deposits and fee payments? // TODO: Add in some concurrency (and measure how much we need it). -use crate::config::{ - utxo::{ - AssetId, AssetValue, Checkpoint, FullIncomingNote, MerkleTreeConfiguration, Parameters, +use crate::{ + config::{ + utxo::{ + AssetId, AssetValue, Checkpoint, FullIncomingNote, MerkleTreeConfiguration, Parameters, + }, + AccountId, Config, MultiVerifyingContext, Nullifier, ProofSystem, TransferPost, Utxo, + UtxoAccumulatorModel, }, - AccountId, Config, MultiVerifyingContext, Nullifier, ProofSystem, TransferPost, Utxo, - UtxoAccumulatorModel, + signer::InitialSyncData, }; use alloc::{sync::Arc, vec::Vec}; use core::convert::Infallible; @@ -50,7 +53,7 @@ use manta_crypto::{ constraint::ProofSystem as _, merkle_tree::{ self, - forest::{Configuration, FixedIndex}, + forest::{Configuration, FixedIndex, Forest}, }, }; use manta_util::future::{LocalBoxFuture, LocalBoxFutureResult}; @@ -214,6 +217,46 @@ impl Ledger { } true } + + /// + #[inline] + pub fn initial_read(&self) -> ReadResponse { + let mut utxos = Vec::new(); + for (i, mut index) in (0..MerkleTreeConfiguration::FOREST_WIDTH).enumerate() { + let shard = &self.shards[&MerkleForestIndex::from_index(i)]; + while let Some(entry) = shard.get_index(index) { + utxos.push(entry.0.clone()); + index += 1; + } + } + let senders = self.nullifiers.iter().cloned().collect::>(); + let utxo_merkle_forest = self.utxo_forest.clone(); + let mut membership_proof_data = Vec::new(); + for index in (0..MerkleTreeConfiguration::FOREST_WIDTH) { + membership_proof_data.push( + utxo_merkle_forest + .forest + .get(index as u8) + .current_path() + .clone() + .into(), + ) + } + ReadResponse { + should_continue: false, + data: InitialSyncData { + utxo_data: utxos, + nullifier_data: senders, + membership_proof_data, + }, + } + } + + /// + #[inline] + pub fn utxos(&self) -> &HashSet { + &self.utxos + } } /// Sender Ledger Error @@ -637,6 +680,19 @@ impl ledger::Read> for LedgerConnection { } } +impl ledger::Read for LedgerConnection { + type Checkpoint = Checkpoint; + + #[inline] + fn read<'s>( + &'s mut self, + checkpoint: &'s Self::Checkpoint, + ) -> LocalBoxFutureResult<'s, ReadResponse, Self::Error> { + let _ = checkpoint; + Box::pin(async move { Ok(self.ledger.read().await.initial_read()) }) + } +} + impl ledger::Write> for LedgerConnection { type Response = bool; diff --git a/manta-pay/src/simulation/mod.rs b/manta-pay/src/simulation/mod.rs index fb3b222e4..5f4523b0c 100644 --- a/manta-pay/src/simulation/mod.rs +++ b/manta-pay/src/simulation/mod.rs @@ -23,17 +23,18 @@ use crate::{ UtxoAccumulatorModel, }, key::KeySecret, - signer::{base::Signer, functions}, + signer::{base::Signer, functions, InitialSyncData}, simulation::ledger::{Ledger, LedgerConnection}, }; use alloc::{format, sync::Arc}; -use core::fmt::Debug; +use core::{fmt::Debug, ops::Deref}; use manta_accounting::{ self, asset::AssetList, key::AccountTable, wallet::{ self, + signer::SyncData, test::{self, PublicBalanceOracle}, Error, }, @@ -139,11 +140,18 @@ impl Simulation { self.setup(&mut ledger); let ledger = Arc::new(RwLock::new(ledger)); self.run_with( - move |i| LedgerConnection::new(account_id_from_u64(i as u64), ledger.clone()), + |i| LedgerConnection::new(account_id_from_u64(i as u64), ledger.clone()), + |_| sample_signer(proving_context, parameters, utxo_accumulator_model, rng), + move |i| account_id_from_u64(i as u64), + ) + .await; + println!("{:?}", ledger.read().await.utxos()); + self.run_with( + |i| LedgerConnection::new(account_id_from_u64(i as u64), ledger.clone()), move |_| sample_signer(proving_context, parameters, utxo_accumulator_model, rng), move |i| account_id_from_u64(i as u64), ) - .await + .await; } /// Runs the simulation with the given ledger connections and signer connections. @@ -155,8 +163,16 @@ impl Simulation { #[inline] pub async fn run_with(&self, ledger: GL, signer: GS, public_account: GP) where - L: wallet::test::Ledger + PublicBalanceOracle, - S: wallet::signer::Connection, + L: wallet::test::Ledger + + PublicBalanceOracle + + wallet::ledger::Read< + InitialSyncData, + Checkpoint = >>::Checkpoint, + >, + S: wallet::signer::Connection< + Config, + Checkpoint = >>::Checkpoint, + >, S::Error: Debug, GL: FnMut(usize) -> L, GS: FnMut(usize) -> S, From 0c78f264e4ec46e9060f9cb0413bf463b371bb90 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Thu, 16 Mar 2023 20:05:40 +0100 Subject: [PATCH 10/58] change dep --- manta-accounting/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/manta-accounting/Cargo.toml b/manta-accounting/Cargo.toml index 0150fc67a..d2c4a000b 100644 --- a/manta-accounting/Cargo.toml +++ b/manta-accounting/Cargo.toml @@ -48,6 +48,7 @@ test = [ "futures", "indexmap", "manta-crypto/arkworks", + "manta_crypto/getrandom", "manta-crypto/rand", "manta-crypto/test", "parking_lot", From be369ad40668f7259cd039c4355a3fc79680a883 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Thu, 16 Mar 2023 20:10:21 +0100 Subject: [PATCH 11/58] fixed dep --- manta-accounting/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manta-accounting/Cargo.toml b/manta-accounting/Cargo.toml index d2c4a000b..285e85010 100644 --- a/manta-accounting/Cargo.toml +++ b/manta-accounting/Cargo.toml @@ -48,7 +48,7 @@ test = [ "futures", "indexmap", "manta-crypto/arkworks", - "manta_crypto/getrandom", + "manta-crypto/getrandom", "manta-crypto/rand", "manta-crypto/test", "parking_lot", From 805325869976db28487fb9ef248d329e794b7390 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Thu, 16 Mar 2023 20:15:56 +0100 Subject: [PATCH 12/58] dep --- manta-crypto/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manta-crypto/Cargo.toml b/manta-crypto/Cargo.toml index d8ad63cec..29b45adc3 100644 --- a/manta-crypto/Cargo.toml +++ b/manta-crypto/Cargo.toml @@ -70,7 +70,7 @@ std = [ ] # Testing Frameworks -test = [] +test = ["getrandom"] [dependencies] ark-bls12-381 = { version = "0.3.0", optional = true, default-features = false, features = ["curve"] } From ca0ad1d922f479885539a189254572b7e20f8a9d Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Fri, 17 Mar 2023 13:49:49 +0100 Subject: [PATCH 13/58] rearranged tests --- manta-crypto/src/merkle_tree/inner_tree.rs | 2 +- manta-crypto/src/merkle_tree/test.rs | 469 --------------------- manta-pay/src/simulation/ledger/mod.rs | 4 +- manta-trusted-setup/src/groth16/mpc.rs | 1 + 4 files changed, 4 insertions(+), 472 deletions(-) delete mode 100644 manta-crypto/src/merkle_tree/test.rs diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index 7ab7dddd0..42eca5012 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -25,7 +25,7 @@ use crate::merkle_tree::{ path_length, Configuration, InnerDigest, Node, Parameters, Parity, }; use alloc::collections::btree_map; -use core::{default, fmt::Debug, hash::Hash, iter::FusedIterator, marker::PhantomData, ops::Index}; +use core::{fmt::Debug, hash::Hash, iter::FusedIterator, marker::PhantomData, ops::Index}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; diff --git a/manta-crypto/src/merkle_tree/test.rs b/manta-crypto/src/merkle_tree/test.rs deleted file mode 100644 index f80e9ec27..000000000 --- a/manta-crypto/src/merkle_tree/test.rs +++ /dev/null @@ -1,469 +0,0 @@ -// Copyright 2019-2022 Manta Network. -// This file is part of manta-rs. -// -// manta-rs is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// manta-rs is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with manta-rs. If not, see . - -//! Testing Framework - -use crate::{ - accumulator::{Accumulator, FromItemsAndWitnesses}, - merkle_tree::{ - forest::{self, Forest, MerkleForest, TreeArrayMerkleForest}, - fork::ForkedTree, - full::{Full, FullMerkleTree}, - partial::{Partial, PartialMerkleTree}, - Configuration, HashConfiguration, IdentityLeafHash, InnerDigest, InnerHash, - InnerHashParameters, Leaf, LeafHashParameters, MerkleTree, Parameters, Path, Tree, - WithProofs, - }, - rand::{OsRng, Rand, RngCore, Sample}, -}; -use alloc::string::String; -use core::{fmt::Debug, hash::Hash, marker::PhantomData}; - -use super::forest::FixedIndex; - -/// Hash Parameter Sampling -pub trait HashParameterSampling: HashConfiguration { - /// Leaf Hash Parameter Distribution - type LeafHashParameterDistribution; - - /// Inner Hash Parameter Distribution - type InnerHashParameterDistribution; - - /// Sample leaf hash parameters from `distribution` using the given `rng`. - fn sample_leaf_hash_parameters( - distribution: Self::LeafHashParameterDistribution, - rng: &mut R, - ) -> LeafHashParameters - where - R: RngCore + ?Sized; - - /// Sample inner hash parameters from `distribution` using the given `rng`. - fn sample_inner_hash_parameters( - distribution: Self::InnerHashParameterDistribution, - rng: &mut R, - ) -> InnerHashParameters - where - R: RngCore + ?Sized; -} - -/// Hash Parameter Distribution -#[derive(derivative::Derivative)] -#[derivative( - Clone( - bound = "C::LeafHashParameterDistribution: Clone, C::InnerHashParameterDistribution: Clone" - ), - Copy( - bound = "C::LeafHashParameterDistribution: Copy, C::InnerHashParameterDistribution: Copy" - ), - Debug( - bound = "C::LeafHashParameterDistribution: Debug, C::InnerHashParameterDistribution: Debug" - ), - Default( - bound = "C::LeafHashParameterDistribution: Default, C::InnerHashParameterDistribution: Default" - ), - Eq(bound = "C::LeafHashParameterDistribution: Eq, C::InnerHashParameterDistribution: Eq"), - Hash( - bound = "C::LeafHashParameterDistribution: Hash, C::InnerHashParameterDistribution: Hash" - ), - PartialEq(bound = "C::LeafHashParameterDistribution: PartialEq, - C::InnerHashParameterDistribution: PartialEq") -)] -pub struct HashParameterDistribution -where - C: HashParameterSampling + ?Sized, -{ - /// Leaf Hash Parameter Distribution - pub leaf: C::LeafHashParameterDistribution, - - /// Inner Hash Parameter Distribution - pub inner: C::InnerHashParameterDistribution, -} - -impl Sample> for Parameters -where - C: HashParameterSampling + ?Sized, -{ - #[inline] - fn sample(distribution: HashParameterDistribution, rng: &mut R) -> Self - where - R: RngCore + ?Sized, - { - Self::new( - C::sample_leaf_hash_parameters(distribution.leaf, rng), - C::sample_inner_hash_parameters(distribution.inner, rng), - ) - } -} - -/// Tests that a tree constructed with `parameters` can accept at least two leaves without -/// failing. -#[inline] -pub fn push_twice_to_empty_tree_succeeds( - parameters: Parameters, - lhs: &Leaf, - rhs: &Leaf, -) -> Parameters -where - C: Configuration + ?Sized, - T: Tree, -{ - let mut tree = MerkleTree::::new(parameters); - assert!( - tree.push(lhs), - "Trees always have a capacity of at least two." - ); - assert!( - tree.push(rhs), - "Trees always have a capacity of at least two." - ); - tree.into_parameters() -} - -/// Tests path construction by checking that the path at the given `index` on `tree` is a valid -/// [`Path`](super::Path) for `leaf`. -#[inline] -pub fn assert_valid_path(tree: &MerkleTree, index: usize, leaf: &Leaf) -where - C: Configuration + ?Sized, - T: Tree + WithProofs, - InnerDigest: Debug + PartialEq, - Path: Debug, -{ - let path = tree.path(index).expect("Only valid queries are accepted."); - let root = tree.root(); - assert!( - path.verify(tree.parameters(), root, leaf), - "Path returned from tree was not valid: {:?}. Expected {:?} but got {:?}.", - path, - root, - path.root(&tree.parameters, &tree.parameters.digest(leaf)), - ); -} - -/// Tests path construction for multiple insertions. This is an extension of the -/// [`assert_valid_path`] test. -#[inline] -pub fn assert_valid_paths(tree: &mut MerkleTree, leaves: &[Leaf]) -where - C: Configuration + ?Sized, - T: Tree + WithProofs, - InnerDigest: Debug + PartialEq, - Path: Debug, - Leaf: Sized, -{ - let starting_index = tree.len(); - for (i, leaf) in leaves.iter().enumerate() { - tree.push(leaf); - for (j, previous_leaf) in leaves.iter().enumerate().take(i + 1) { - assert_valid_path(tree, starting_index + j, previous_leaf); - } - } -} - -/// Test Inner Hash -/// -/// # Warning -/// -/// This is only meant for testing purposes, and should not be used in any production or -/// cryptographically secure environments. -pub trait TestHash { - /// Joins `lhs` and `rhs` into an output hash value. - fn join(lhs: &Self, rhs: &Self) -> Self; -} - -impl TestHash for u64 { - #[inline] - fn join(lhs: &Self, rhs: &Self) -> Self { - *lhs ^ *rhs - } -} - -impl TestHash for String { - #[inline] - fn join(lhs: &Self, rhs: &Self) -> Self { - let mut lhs = lhs.clone(); - lhs.push_str(rhs); - lhs - } -} - -/// Test Merkle Tree Configuration -/// -/// # Warning -/// -/// This is only meant for testing purposes, and should not be used in production or -/// cryptographically secure environments. -#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct Test(PhantomData) -where - T: Clone + Default + PartialEq + TestHash; - -impl InnerHash for Test -where - T: Clone + Default + PartialEq + TestHash, -{ - type LeafDigest = T; - type Parameters = (); - type Output = T; - - #[inline] - fn join( - parameters: &Self::Parameters, - lhs: &Self::Output, - rhs: &Self::Output, - _: &mut (), - ) -> Self::Output { - let _ = parameters; - TestHash::join(lhs, rhs) - } - - #[inline] - fn join_leaves( - parameters: &Self::Parameters, - lhs: &Self::LeafDigest, - rhs: &Self::LeafDigest, - _: &mut (), - ) -> Self::Output { - let _ = parameters; - TestHash::join(lhs, rhs) - } -} - -impl HashConfiguration for Test -where - T: Clone + Default + PartialEq + TestHash, -{ - type LeafHash = IdentityLeafHash; - type InnerHash = Test; -} - -impl Configuration for Test -where - T: Clone + Default + PartialEq + TestHash, -{ - const HEIGHT: usize = HEIGHT; -} - -impl HashParameterSampling for Test -where - T: Clone + Default + PartialEq + TestHash, -{ - type LeafHashParameterDistribution = (); - type InnerHashParameterDistribution = (); - - #[inline] - fn sample_leaf_hash_parameters( - distribution: Self::LeafHashParameterDistribution, - rng: &mut R, - ) -> LeafHashParameters - where - R: RngCore + ?Sized, - { - let _ = (distribution, rng); - } - - #[inline] - fn sample_inner_hash_parameters( - distribution: Self::InnerHashParameterDistribution, - rng: &mut R, - ) -> InnerHashParameters - where - R: RngCore + ?Sized, - { - let _ = (distribution, rng); - } -} - -/// -#[test] -fn test_from_leaves_and_path() { - let mut rng = OsRng; - const HEIGHT: usize = 7; - type Config = Test; - let parameters = Parameters::::sample(Default::default(), &mut rng); - let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); - let inner_element_index = rng.gen_range(0..number_of_insertions - 3); // make sure this one isn't the last nor its sibling - println!("{number_of_insertions}, {inner_element_index}"); - let mut tree = FullMerkleTree::::new(parameters); - let mut insertions = Vec::::with_capacity(number_of_insertions); - for _ in 0..number_of_insertions { - insertions.push(rng.gen()); - } - for leaf in &insertions { - tree.insert(leaf); - } - let forked_tree = ForkedTree::>::new(tree.tree.clone(), ¶meters); - let path = tree.current_path(); - let partial_tree = PartialMerkleTree::> { - parameters, - tree: Partial::from_leaves_and_path_unchecked( - ¶meters, - insertions.clone(), - path.clone().into(), - ), - }; - let forked_partial_tree = ForkedTree::>::from_leaves_and_path_unchecked( - ¶meters, - insertions.clone(), - path.clone().into(), - ); - let root = tree.root().clone(); - let partial_root = partial_tree.root().clone(); - let forked_root = forked_tree.root().clone(); - let forked_partial_root = forked_partial_tree.root().clone(); - assert_eq!(root, partial_root, "Roots must be equal"); - assert_eq!(root, forked_root, "Roots must be equal"); - assert_eq!(root, forked_partial_root, "Roots must be equal"); - let proof_full_inner = tree - .prove(&insertions[inner_element_index]) - .expect("Failed to generate proof"); - let proof_partial_inner = partial_tree - .prove(&insertions[inner_element_index]) - .expect("Failed to generate proof"); - assert!( - proof_full_inner.verify(¶meters, &insertions[inner_element_index], &mut ()), - "Inner proof in the full tree must be valid" - ); - assert!( - !proof_partial_inner.verify(¶meters, &insertions[inner_element_index], &mut ()), - "Inner proof in the partial tree must be invalid" - ); - let proof_full = tree - .prove(&insertions[number_of_insertions - 1]) - .expect("Failed to generate proof"); - let proof_partial = partial_tree - .prove(&insertions[number_of_insertions - 1]) - .expect("Failed to generate proof"); - assert!( - proof_full.verify(¶meters, &insertions[number_of_insertions - 1], &mut ()), - "Final proof in the full tree must be valid" - ); - assert!( - proof_partial.verify(¶meters, &insertions[number_of_insertions - 1], &mut ()), - "Final proof in the partial tree must be valid" - ); -} - -/// -#[derive(PartialEq)] -pub enum Index { - /// - Zero, - - /// - One, -} - -impl From for usize { - fn from(value: Index) -> Self { - match value { - Index::Zero => 0, - Index::One => 1, - } - } -} - -impl FixedIndex<2> for Index { - fn from_index(index: usize) -> Self { - if index % 2 == 0 { - Index::Zero - } else { - Index::One - } - } -} - -impl forest::Configuration for Test { - type Index = Index; - fn tree_index(leaf: &Leaf) -> Self::Index { - let parity = leaf % 2; - if parity == 0 { - Index::Zero - } else { - Index::One - } - } -} -/// -#[test] -fn test_from_leaves_and_path_forest() { - let mut rng = OsRng; - const HEIGHT: usize = 7; - type Config = Test; - let parameters = Parameters::::sample(Default::default(), &mut rng); - let mut forest = - TreeArrayMerkleForest::>, 2>::new(parameters); - let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); - let mut insertions = Vec::::with_capacity(number_of_insertions); - for _ in 0..number_of_insertions { - insertions.push(rng.gen()); - } - for leaf in &insertions { - forest.insert(leaf); - } - let path_1 = Path::from(forest.forest.get(Index::Zero).current_path()); - let path_2 = Path::from(forest.forest.get(Index::One).current_path()); - let paths = vec![path_1, path_2]; - let partial_forest = - TreeArrayMerkleForest::<_, ForkedTree<_, Partial>, 2>::from_items_and_witnesses( - ¶meters, - insertions.clone(), - paths, - ); - for leaf in &insertions { - assert_eq!(forest.output_from(leaf), partial_forest.output_from(leaf)); - } -} - -/// -#[test] -fn visual_test_with_strings() { - const HEIGHT: usize = 5; - let mut rng = OsRng; - let parameters = Parameters::>::sample(Default::default(), &mut rng); - let mut tree = FullMerkleTree::>::new(parameters); - let insertions = vec![ - "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", - ] - .into_iter() - .map(String::from) - .collect::>(); - for leaf in &insertions { - tree.insert(leaf); - } - let root = tree.root().clone(); - let path = tree.current_path(); - println!("{path:?}"); - let partial_tree = PartialMerkleTree::> { - parameters, - tree: Partial::from_leaves_and_path_unchecked(¶meters, insertions.clone(), path.into()), - }; - let second_root = partial_tree.root().clone(); - assert_eq!(root, second_root); - const INNER_ELEMENT_INDEX: usize = 10; // k - let proof = tree.prove(&insertions[INNER_ELEMENT_INDEX]).unwrap(); - println!("{proof:?}"); - let proof_2 = partial_tree - .prove(&insertions[INNER_ELEMENT_INDEX]) - .unwrap(); - println!("{proof_2:?}"); - let proof_3 = tree.prove(&insertions[insertions.len() - 1]).unwrap(); - let proof_4 = partial_tree - .prove(&insertions[insertions.len() - 1]) - .unwrap(); - println!("{proof_3:?}"); - println!("{proof_4:?}"); -} diff --git a/manta-pay/src/simulation/ledger/mod.rs b/manta-pay/src/simulation/ledger/mod.rs index d84395c80..a6a19942c 100644 --- a/manta-pay/src/simulation/ledger/mod.rs +++ b/manta-pay/src/simulation/ledger/mod.rs @@ -225,14 +225,14 @@ impl Ledger { for (i, mut index) in (0..MerkleTreeConfiguration::FOREST_WIDTH).enumerate() { let shard = &self.shards[&MerkleForestIndex::from_index(i)]; while let Some(entry) = shard.get_index(index) { - utxos.push(entry.0.clone()); + utxos.push(entry.0); index += 1; } } let senders = self.nullifiers.iter().cloned().collect::>(); let utxo_merkle_forest = self.utxo_forest.clone(); let mut membership_proof_data = Vec::new(); - for index in (0..MerkleTreeConfiguration::FOREST_WIDTH) { + for index in 0..MerkleTreeConfiguration::FOREST_WIDTH { membership_proof_data.push( utxo_merkle_forest .forest diff --git a/manta-trusted-setup/src/groth16/mpc.rs b/manta-trusted-setup/src/groth16/mpc.rs index abe445825..29c5aab15 100644 --- a/manta-trusted-setup/src/groth16/mpc.rs +++ b/manta-trusted-setup/src/groth16/mpc.rs @@ -513,6 +513,7 @@ where /// transitions. /// /// [`Challenge`]: mpc::ChallengeType::Challenge +#[allow(clippy::redundant_clone)] // The state clone is actually necessary here. #[inline] pub fn verify_transform_all( mut challenge: C::Challenge, From 9cc2e1fa2b348ce0be151250e8e6bda1714761e9 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Fri, 17 Mar 2023 13:52:54 +0100 Subject: [PATCH 14/58] restored cargo.toml --- manta-accounting/Cargo.toml | 1 - manta-crypto/Cargo.toml | 2 +- manta-crypto/src/merkle_tree/test/mod.rs | 327 +++++++++++++++++++ manta-crypto/src/merkle_tree/test/partial.rs | 134 ++++++++ 4 files changed, 462 insertions(+), 2 deletions(-) create mode 100644 manta-crypto/src/merkle_tree/test/mod.rs create mode 100644 manta-crypto/src/merkle_tree/test/partial.rs diff --git a/manta-accounting/Cargo.toml b/manta-accounting/Cargo.toml index 285e85010..0150fc67a 100644 --- a/manta-accounting/Cargo.toml +++ b/manta-accounting/Cargo.toml @@ -48,7 +48,6 @@ test = [ "futures", "indexmap", "manta-crypto/arkworks", - "manta-crypto/getrandom", "manta-crypto/rand", "manta-crypto/test", "parking_lot", diff --git a/manta-crypto/Cargo.toml b/manta-crypto/Cargo.toml index 29b45adc3..d8ad63cec 100644 --- a/manta-crypto/Cargo.toml +++ b/manta-crypto/Cargo.toml @@ -70,7 +70,7 @@ std = [ ] # Testing Frameworks -test = ["getrandom"] +test = [] [dependencies] ark-bls12-381 = { version = "0.3.0", optional = true, default-features = false, features = ["curve"] } diff --git a/manta-crypto/src/merkle_tree/test/mod.rs b/manta-crypto/src/merkle_tree/test/mod.rs new file mode 100644 index 000000000..e01c4d75b --- /dev/null +++ b/manta-crypto/src/merkle_tree/test/mod.rs @@ -0,0 +1,327 @@ +// Copyright 2019-2022 Manta Network. +// This file is part of manta-rs. +// +// manta-rs is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// manta-rs is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with manta-rs. If not, see . + +//! Testing Framework + +use crate::{ + merkle_tree::{ + forest, Configuration, HashConfiguration, IdentityLeafHash, InnerDigest, InnerHash, + InnerHashParameters, Leaf, LeafHashParameters, MerkleTree, Parameters, Path, Tree, + WithProofs, + }, + rand::{RngCore, Sample}, +}; +use alloc::string::String; +use core::{fmt::Debug, hash::Hash, marker::PhantomData}; + +#[cfg(test)] +pub mod partial; + +/// Hash Parameter Sampling +pub trait HashParameterSampling: HashConfiguration { + /// Leaf Hash Parameter Distribution + type LeafHashParameterDistribution; + + /// Inner Hash Parameter Distribution + type InnerHashParameterDistribution; + + /// Sample leaf hash parameters from `distribution` using the given `rng`. + fn sample_leaf_hash_parameters( + distribution: Self::LeafHashParameterDistribution, + rng: &mut R, + ) -> LeafHashParameters + where + R: RngCore + ?Sized; + + /// Sample inner hash parameters from `distribution` using the given `rng`. + fn sample_inner_hash_parameters( + distribution: Self::InnerHashParameterDistribution, + rng: &mut R, + ) -> InnerHashParameters + where + R: RngCore + ?Sized; +} + +/// Hash Parameter Distribution +#[derive(derivative::Derivative)] +#[derivative( + Clone( + bound = "C::LeafHashParameterDistribution: Clone, C::InnerHashParameterDistribution: Clone" + ), + Copy( + bound = "C::LeafHashParameterDistribution: Copy, C::InnerHashParameterDistribution: Copy" + ), + Debug( + bound = "C::LeafHashParameterDistribution: Debug, C::InnerHashParameterDistribution: Debug" + ), + Default( + bound = "C::LeafHashParameterDistribution: Default, C::InnerHashParameterDistribution: Default" + ), + Eq(bound = "C::LeafHashParameterDistribution: Eq, C::InnerHashParameterDistribution: Eq"), + Hash( + bound = "C::LeafHashParameterDistribution: Hash, C::InnerHashParameterDistribution: Hash" + ), + PartialEq(bound = "C::LeafHashParameterDistribution: PartialEq, + C::InnerHashParameterDistribution: PartialEq") +)] +pub struct HashParameterDistribution +where + C: HashParameterSampling + ?Sized, +{ + /// Leaf Hash Parameter Distribution + pub leaf: C::LeafHashParameterDistribution, + + /// Inner Hash Parameter Distribution + pub inner: C::InnerHashParameterDistribution, +} + +impl Sample> for Parameters +where + C: HashParameterSampling + ?Sized, +{ + #[inline] + fn sample(distribution: HashParameterDistribution, rng: &mut R) -> Self + where + R: RngCore + ?Sized, + { + Self::new( + C::sample_leaf_hash_parameters(distribution.leaf, rng), + C::sample_inner_hash_parameters(distribution.inner, rng), + ) + } +} + +/// Tests that a tree constructed with `parameters` can accept at least two leaves without +/// failing. +#[inline] +pub fn push_twice_to_empty_tree_succeeds( + parameters: Parameters, + lhs: &Leaf, + rhs: &Leaf, +) -> Parameters +where + C: Configuration + ?Sized, + T: Tree, +{ + let mut tree = MerkleTree::::new(parameters); + assert!( + tree.push(lhs), + "Trees always have a capacity of at least two." + ); + assert!( + tree.push(rhs), + "Trees always have a capacity of at least two." + ); + tree.into_parameters() +} + +/// Tests path construction by checking that the path at the given `index` on `tree` is a valid +/// [`Path`](super::Path) for `leaf`. +#[inline] +pub fn assert_valid_path(tree: &MerkleTree, index: usize, leaf: &Leaf) +where + C: Configuration + ?Sized, + T: Tree + WithProofs, + InnerDigest: Debug + PartialEq, + Path: Debug, +{ + let path = tree.path(index).expect("Only valid queries are accepted."); + let root = tree.root(); + assert!( + path.verify(tree.parameters(), root, leaf), + "Path returned from tree was not valid: {:?}. Expected {:?} but got {:?}.", + path, + root, + path.root(&tree.parameters, &tree.parameters.digest(leaf)), + ); +} + +/// Tests path construction for multiple insertions. This is an extension of the +/// [`assert_valid_path`] test. +#[inline] +pub fn assert_valid_paths(tree: &mut MerkleTree, leaves: &[Leaf]) +where + C: Configuration + ?Sized, + T: Tree + WithProofs, + InnerDigest: Debug + PartialEq, + Path: Debug, + Leaf: Sized, +{ + let starting_index = tree.len(); + for (i, leaf) in leaves.iter().enumerate() { + tree.push(leaf); + for (j, previous_leaf) in leaves.iter().enumerate().take(i + 1) { + assert_valid_path(tree, starting_index + j, previous_leaf); + } + } +} + +/// Test Inner Hash +/// +/// # Warning +/// +/// This is only meant for testing purposes, and should not be used in any production or +/// cryptographically secure environments. +pub trait TestHash { + /// Joins `lhs` and `rhs` into an output hash value. + fn join(lhs: &Self, rhs: &Self) -> Self; +} + +impl TestHash for u64 { + #[inline] + fn join(lhs: &Self, rhs: &Self) -> Self { + *lhs ^ *rhs + } +} + +impl TestHash for String { + #[inline] + fn join(lhs: &Self, rhs: &Self) -> Self { + let mut lhs = lhs.clone(); + lhs.push_str(rhs); + lhs + } +} + +/// Test Merkle Tree Configuration +/// +/// # Warning +/// +/// This is only meant for testing purposes, and should not be used in production or +/// cryptographically secure environments. +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Test(PhantomData) +where + T: Clone + Default + PartialEq + TestHash; + +impl InnerHash for Test +where + T: Clone + Default + PartialEq + TestHash, +{ + type LeafDigest = T; + type Parameters = (); + type Output = T; + + #[inline] + fn join( + parameters: &Self::Parameters, + lhs: &Self::Output, + rhs: &Self::Output, + _: &mut (), + ) -> Self::Output { + let _ = parameters; + TestHash::join(lhs, rhs) + } + + #[inline] + fn join_leaves( + parameters: &Self::Parameters, + lhs: &Self::LeafDigest, + rhs: &Self::LeafDigest, + _: &mut (), + ) -> Self::Output { + let _ = parameters; + TestHash::join(lhs, rhs) + } +} + +impl HashConfiguration for Test +where + T: Clone + Default + PartialEq + TestHash, +{ + type LeafHash = IdentityLeafHash; + type InnerHash = Test; +} + +impl Configuration for Test +where + T: Clone + Default + PartialEq + TestHash, +{ + const HEIGHT: usize = HEIGHT; +} + +impl HashParameterSampling for Test +where + T: Clone + Default + PartialEq + TestHash, +{ + type LeafHashParameterDistribution = (); + type InnerHashParameterDistribution = (); + + #[inline] + fn sample_leaf_hash_parameters( + distribution: Self::LeafHashParameterDistribution, + rng: &mut R, + ) -> LeafHashParameters + where + R: RngCore + ?Sized, + { + let _ = (distribution, rng); + } + + #[inline] + fn sample_inner_hash_parameters( + distribution: Self::InnerHashParameterDistribution, + rng: &mut R, + ) -> InnerHashParameters + where + R: RngCore + ?Sized, + { + let _ = (distribution, rng); + } +} + +/// Binary Index +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum BinaryIndex { + /// Zero + Zero, + + /// One + One, +} + +impl From for usize { + fn from(value: BinaryIndex) -> Self { + match value { + BinaryIndex::Zero => 0, + BinaryIndex::One => 1, + } + } +} + +impl forest::FixedIndex<2> for BinaryIndex { + fn from_index(index: usize) -> Self { + if index % 2 == 0 { + BinaryIndex::Zero + } else { + BinaryIndex::One + } + } +} + +impl forest::Configuration for Test { + type Index = BinaryIndex; + + #[inline] + fn tree_index(leaf: &Leaf) -> Self::Index { + let parity = leaf % 2; + if parity == 0 { + BinaryIndex::Zero + } else { + BinaryIndex::One + } + } +} diff --git a/manta-crypto/src/merkle_tree/test/partial.rs b/manta-crypto/src/merkle_tree/test/partial.rs new file mode 100644 index 000000000..cf04639ee --- /dev/null +++ b/manta-crypto/src/merkle_tree/test/partial.rs @@ -0,0 +1,134 @@ +// Copyright 2019-2022 Manta Network. +// This file is part of manta-rs. +// +// manta-rs is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// manta-rs is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with manta-rs. If not, see . + +//! Partial Merkle Tree + +use crate::{ + accumulator::{Accumulator, FromItemsAndWitnesses}, + merkle_tree::{ + forest::{Forest, TreeArrayMerkleForest}, + fork::ForkedTree, + full::{Full, FullMerkleTree}, + partial::{Partial, PartialMerkleTree}, + path::Path, + test::{BinaryIndex, Test}, + tree::Parameters, + }, + rand::{OsRng, Rand, Sample}, +}; + +/// Tests the [`Partial`] tree generated from a set of leaves and a [`Path`] behaves +/// as expected. +#[test] +fn test_from_leaves_and_path() { + let mut rng = OsRng; + const HEIGHT: usize = 7; + type Config = Test; + let parameters = Parameters::::sample(Default::default(), &mut rng); + let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); + let inner_element_index = rng.gen_range(0..number_of_insertions - 3); + println!("{number_of_insertions}, {inner_element_index}"); + let mut tree = FullMerkleTree::::new(parameters); + let mut insertions = Vec::::with_capacity(number_of_insertions); + for _ in 0..number_of_insertions { + insertions.push(rng.gen()); + } + for leaf in &insertions { + tree.insert(leaf); + } + let forked_tree = ForkedTree::>::new(tree.tree.clone(), ¶meters); + let path = tree.current_path(); + let partial_tree = PartialMerkleTree::> { + parameters, + tree: Partial::from_leaves_and_path_unchecked( + ¶meters, + insertions.clone(), + path.clone().into(), + ), + }; + let forked_partial_tree = ForkedTree::>::from_leaves_and_path_unchecked( + ¶meters, + insertions.clone(), + path.into(), + ); + let root = tree.root(); + let partial_root = partial_tree.root(); + let forked_root = forked_tree.root(); + let forked_partial_root = forked_partial_tree.root(); + assert_eq!(root, partial_root, "Roots must be equal"); + assert_eq!(root, forked_root, "Roots must be equal"); + assert_eq!(root, forked_partial_root, "Roots must be equal"); + let proof_full_inner = tree + .prove(&insertions[inner_element_index]) + .expect("Failed to generate proof"); + let proof_partial_inner = partial_tree + .prove(&insertions[inner_element_index]) + .expect("Failed to generate proof"); + assert!( + proof_full_inner.verify(¶meters, &insertions[inner_element_index], &mut ()), + "Inner proof in the full tree must be valid" + ); + assert!( + !proof_partial_inner.verify(¶meters, &insertions[inner_element_index], &mut ()), + "Inner proof in the partial tree must be invalid" + ); + let proof_full = tree + .prove(&insertions[number_of_insertions - 1]) + .expect("Failed to generate proof"); + let proof_partial = partial_tree + .prove(&insertions[number_of_insertions - 1]) + .expect("Failed to generate proof"); + assert!( + proof_full.verify(¶meters, &insertions[number_of_insertions - 1], &mut ()), + "Final proof in the full tree must be valid" + ); + assert!( + proof_partial.verify(¶meters, &insertions[number_of_insertions - 1], &mut ()), + "Final proof in the partial tree must be valid" + ); +} + +/// Tests the forest consisting of [`Partial`] trees generated from a set of leaves +/// and a [`Path`]s behaves as expected. +#[test] +fn test_from_leaves_and_path_forest() { + let mut rng = OsRng; + const HEIGHT: usize = 7; + type Config = Test; + let parameters = Parameters::::sample(Default::default(), &mut rng); + let mut forest = + TreeArrayMerkleForest::>, 2>::new(parameters); + let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); + let mut insertions = Vec::::with_capacity(number_of_insertions); + for _ in 0..number_of_insertions { + insertions.push(rng.gen()); + } + for leaf in &insertions { + forest.insert(leaf); + } + let path_1 = Path::from(forest.forest.get(BinaryIndex::Zero).current_path()); + let path_2 = Path::from(forest.forest.get(BinaryIndex::One).current_path()); + let paths = vec![path_1, path_2]; + let partial_forest = + TreeArrayMerkleForest::<_, ForkedTree<_, Partial>, 2>::from_items_and_witnesses( + ¶meters, + insertions.clone(), + paths, + ); + for leaf in &insertions { + assert_eq!(forest.output_from(leaf), partial_forest.output_from(leaf)); + } +} From 68ce7ac8a48fce8098e1bc7aeed73c26a6d8ddb4 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Fri, 17 Mar 2023 13:54:07 +0100 Subject: [PATCH 15/58] clippy --- manta-pay/src/simulation/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manta-pay/src/simulation/mod.rs b/manta-pay/src/simulation/mod.rs index 5f4523b0c..e11dc845b 100644 --- a/manta-pay/src/simulation/mod.rs +++ b/manta-pay/src/simulation/mod.rs @@ -27,7 +27,7 @@ use crate::{ simulation::ledger::{Ledger, LedgerConnection}, }; use alloc::{format, sync::Arc}; -use core::{fmt::Debug, ops::Deref}; +use core::{fmt::Debug}; use manta_accounting::{ self, asset::AssetList, From d9c2a77801038f3ad39f009e75a79359ffb8717b Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Fri, 17 Mar 2023 14:29:52 +0100 Subject: [PATCH 16/58] changed nullifier data and batched the initial sync --- manta-accounting/src/wallet/mod.rs | 23 ++++++++++++------- .../src/wallet/signer/functions.rs | 8 +++---- manta-accounting/src/wallet/signer/mod.rs | 6 ++--- manta-pay/src/simulation/ledger/mod.rs | 2 +- manta-pay/src/simulation/mod.rs | 2 +- 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 8b4138ed4..88cab5b03 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -268,11 +268,21 @@ where /// #[inline] pub async fn initial_sync(&mut self) -> Result<(), Error> + where + L: ledger::Read, Checkpoint = S::Checkpoint>, + { + while self.initial_sync_partial().await?.is_continue() {} + Ok(()) + } + + /// + #[inline] + pub async fn initial_sync_partial(&mut self) -> Result> where L: ledger::Read, Checkpoint = S::Checkpoint>, { let ReadResponse { - should_continue: _, + should_continue, data, } = self .ledger @@ -280,7 +290,7 @@ where .await .map_err(Error::LedgerConnectionError)?; self.signer_initial_sync(data).await?; - Ok(()) + Ok(ControlFlow::should_continue(should_continue)) } /// Pulls data from the ledger, synchronizing the wallet and balance state. This method returns @@ -385,16 +395,13 @@ where balance_update, }) => { match balance_update { - BalanceUpdate::Partial { deposit, withdraw } => { - self.assets.deposit_all(deposit); - if !self.assets.withdraw_all(withdraw) { - return Err(Error::Inconsistency(InconsistencyError::WalletBalance)); - } - } BalanceUpdate::Full { assets } => { self.assets.clear(); self.assets.deposit_all(assets); } + _ => { + unreachable!("No transactions could have happened on a new account."); + } } self.checkpoint = checkpoint; Ok(()) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 3f5d3663b..018369d12 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -949,7 +949,7 @@ where let InitialSyncData { utxo_data, membership_proof_data, - nullifier_data, + nullifier_count, } = request; let (accumulator, response) = initial_sync_with::( assets, @@ -958,7 +958,7 @@ where ¶meters.parameters, utxo_data, membership_proof_data, - nullifier_data, + nullifier_count, ); *utxo_accumulator = accumulator; utxo_accumulator.commit(); @@ -975,7 +975,7 @@ fn initial_sync_with( parameters: &Parameters, utxos: Vec>, membership_proof_data: Vec>, - nullifiers: Vec>, + nullifier_count: u128, ) -> (C::UtxoAccumulator, SyncResponse) where C: Configuration, @@ -988,7 +988,7 @@ where .collect(), membership_proof_data, ); - checkpoint.update_from_nullifiers(nullifiers.len()); + checkpoint.update_from_nullifiers(nullifier_count as usize); checkpoint.update_from_utxo_accumulator(&accumulator); ( accumulator, diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index fc15209bb..0d90785f6 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -155,11 +155,11 @@ where /// UTXO Data pub utxo_data: Vec>, - /// Nullifier Data - pub nullifier_data: Vec>, - /// Membership Proof Data pub membership_proof_data: Vec>, + + /// Nullifier Count + pub nullifier_count: u128, } /// Signer Synchronization Data diff --git a/manta-pay/src/simulation/ledger/mod.rs b/manta-pay/src/simulation/ledger/mod.rs index a6a19942c..fd30b3c14 100644 --- a/manta-pay/src/simulation/ledger/mod.rs +++ b/manta-pay/src/simulation/ledger/mod.rs @@ -246,8 +246,8 @@ impl Ledger { should_continue: false, data: InitialSyncData { utxo_data: utxos, - nullifier_data: senders, membership_proof_data, + nullifier_count: senders.len() as u128, }, } } diff --git a/manta-pay/src/simulation/mod.rs b/manta-pay/src/simulation/mod.rs index e11dc845b..fffe65bc9 100644 --- a/manta-pay/src/simulation/mod.rs +++ b/manta-pay/src/simulation/mod.rs @@ -27,7 +27,7 @@ use crate::{ simulation::ledger::{Ledger, LedgerConnection}, }; use alloc::{format, sync::Arc}; -use core::{fmt::Debug}; +use core::fmt::Debug; use manta_accounting::{ self, asset::AssetList, From e57d2b6b4296d59d0415b4ad4472de6c69a9d91b Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 20 Mar 2023 15:22:19 +0100 Subject: [PATCH 17/58] final design --- manta-accounting/src/transfer/mod.rs | 1 + manta-accounting/src/wallet/mod.rs | 56 ++++---- .../src/wallet/signer/functions.rs | 24 ++-- manta-accounting/src/wallet/signer/mod.rs | 124 +++++++++++++++++- manta-accounting/src/wallet/test/mod.rs | 3 +- manta-crypto/src/accumulator.rs | 9 +- manta-crypto/src/merkle_tree/forest.rs | 89 ++++++++----- manta-pay/src/signer/base.rs | 5 + manta-pay/src/signer/client/http.rs | 15 ++- manta-pay/src/signer/client/websocket.rs | 15 ++- manta-pay/src/signer/mod.rs | 3 + manta-pay/src/simulation/mod.rs | 3 +- 12 files changed, 256 insertions(+), 91 deletions(-) diff --git a/manta-accounting/src/transfer/mod.rs b/manta-accounting/src/transfer/mod.rs index 0ccc7b6ba..5e6d89da1 100644 --- a/manta-accounting/src/transfer/mod.rs +++ b/manta-accounting/src/transfer/mod.rs @@ -152,6 +152,7 @@ pub trait Configuration { + auth::ProveAuthorization + auth::VerifyAuthorization + auth::DeriveSigningKey + + Clone + for<'a> auth::Sign> + for<'a> auth::VerifySignature> + utxo::AssetType> diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 88cab5b03..97e7a37e4 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -37,9 +37,9 @@ use crate::{ balance::{BTreeMapBalanceState, BalanceState}, ledger::ReadResponse, signer::{ - BalanceUpdate, IdentityRequest, IdentityResponse, InitialSyncData, SignError, - SignRequest, SignResponse, SignWithTransactionDataResponse, SyncData, SyncError, - SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, + BalanceUpdate, Checkpoint, IdentityRequest, IdentityResponse, InitialSyncData, + SignError, SignRequest, SignResponse, SignWithTransactionDataResponse, SyncData, + SyncError, SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, }, }, }; @@ -50,6 +50,8 @@ use manta_util::ops::ControlFlow; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; +use self::signer::InitialSyncRequest; + pub mod balance; pub mod ledger; pub mod signer; @@ -270,29 +272,37 @@ where pub async fn initial_sync(&mut self) -> Result<(), Error> where L: ledger::Read, Checkpoint = S::Checkpoint>, + C: signer::Configuration, + S::Checkpoint: signer::Checkpoint, { - while self.initial_sync_partial().await?.is_continue() {} + let mut is_continue = true; + let mut checkpoint = self.checkpoint.clone(); + let mut request = InitialSyncRequest::::default(); + while is_continue { + let ReadResponse { + should_continue, + data, + } = self + .ledger + .read(&checkpoint) + .await + .map_err(Error::LedgerConnectionError)?; + is_continue = should_continue; + request.extend_with_data( + &self + .signer + .transfer_parameters() + .await + .map_err(Error::SignerConnectionError)?, + data, + ); + checkpoint + .update_from_utxo_count(request.utxo_data.iter().map(|utxos| utxos.len()).collect()) + } + self.signer_initial_sync(request).await?; Ok(()) } - /// - #[inline] - pub async fn initial_sync_partial(&mut self) -> Result> - where - L: ledger::Read, Checkpoint = S::Checkpoint>, - { - let ReadResponse { - should_continue, - data, - } = self - .ledger - .read(&self.checkpoint) - .await - .map_err(Error::LedgerConnectionError)?; - self.signer_initial_sync(data).await?; - Ok(ControlFlow::should_continue(should_continue)) - } - /// Pulls data from the ledger, synchronizing the wallet and balance state. This method returns /// a [`ControlFlow`] for matching against to determine if the wallet requires more /// synchronization. @@ -382,7 +392,7 @@ where #[inline] async fn signer_initial_sync( &mut self, - request: InitialSyncData, + request: InitialSyncRequest, ) -> Result<(), Error> { match self .signer diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 018369d12..3c60b8765 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -35,9 +35,9 @@ use crate::{ UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoAccumulatorWitness, }, wallet::signer::{ - AccountTable, BalanceUpdate, Checkpoint, Configuration, InitialSyncData, SignError, - SignResponse, SignWithTransactionDataResponse, SignWithTransactionDataResult, - SignerParameters, SyncData, SyncError, SyncRequest, SyncResponse, + AccountTable, BalanceUpdate, Checkpoint, Configuration, SignError, SignResponse, + SignWithTransactionDataResponse, SignWithTransactionDataResult, SignerParameters, SyncData, + SyncError, SyncRequest, SyncResponse, }, }; use alloc::{vec, vec::Vec}; @@ -50,6 +50,8 @@ use manta_util::{ vec::VecExt, }; +use super::InitialSyncRequest; + /// Returns the default account for `accounts`. #[inline] pub fn default_account(accounts: &AccountTable) -> Account @@ -110,7 +112,7 @@ where /// Hashes `utxo` using the [`UtxoAccumulatorItemHash`](transfer::Configuration::UtxoAccumulatorItemHash) /// in the transfer [`Configuration`](transfer::Configuration). #[inline] -fn item_hash(parameters: &C::Parameters, utxo: &Utxo) -> UtxoAccumulatorItem +pub fn item_hash(parameters: &C::Parameters, utxo: &Utxo) -> UtxoAccumulatorItem where C: Configuration, { @@ -937,16 +939,15 @@ where /// Updates `assets`, `checkpoint` and `utxo_accumulator`, returning the new asset distribution. #[inline] pub fn intial_sync( - parameters: &SignerParameters, assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator: &mut C::UtxoAccumulator, - request: InitialSyncData, + request: InitialSyncRequest, ) -> Result, SyncError> where C: Configuration, { - let InitialSyncData { + let InitialSyncRequest { utxo_data, membership_proof_data, nullifier_count, @@ -955,7 +956,6 @@ where assets, checkpoint, utxo_accumulator.model(), - ¶meters.parameters, utxo_data, membership_proof_data, nullifier_count, @@ -972,8 +972,7 @@ fn initial_sync_with( assets: &mut C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator_model: &UtxoAccumulatorModel, - parameters: &Parameters, - utxos: Vec>, + utxos: Vec>>, membership_proof_data: Vec>, nullifier_count: u128, ) -> (C::UtxoAccumulator, SyncResponse) @@ -982,10 +981,7 @@ where { let accumulator = C::UtxoAccumulator::from_items_and_witnesses( utxo_accumulator_model, - utxos - .iter() - .map(|utxo| item_hash::(parameters, utxo)) - .collect(), + utxos, membership_proof_data, ); checkpoint.update_from_nullifiers(nullifier_count as usize); diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 0d90785f6..1d5c8325b 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -82,7 +82,7 @@ where /// fn initial_sync( &mut self, - request: InitialSyncData, + request: InitialSyncRequest, ) -> LocalBoxFutureResult, Self::Error>; /// Signs a transaction and returns the ledger transfer posts if successful. @@ -115,6 +115,9 @@ where ) -> LocalBoxFutureResult, Self::Error> where TransferPost: Clone; + + /// + fn transfer_parameters(&mut self) -> LocalBoxFutureResult, Self::Error>; } /// Signer Initial Synchronization Data @@ -162,6 +165,106 @@ where pub nullifier_count: u128, } +/// Signer Initial Synchronization Request +#[cfg_attr( + feature = "serde", + derive(Deserialize, Serialize), + serde( + bound( + deserialize = r" + UtxoAccumulatorItem: Deserialize<'de>, + UtxoAccumulatorWitness: Deserialize<'de>, + Nullifier: Deserialize<'de>, + ", + serialize = r" + UtxoAccumulatorItem: Serialize, + UtxoAccumulatorWitness: Serialize, + Nullifier: Serialize, + ", + ), + crate = "manta_util::serde", + deny_unknown_fields + ) +)] +#[derive(derivative::Derivative)] +#[derivative( + Clone( + bound = "UtxoAccumulatorItem: Clone, UtxoAccumulatorWitness: Clone, Nullifier: Clone" + ), + Debug( + bound = "UtxoAccumulatorItem: Debug, UtxoAccumulatorWitness: Debug, Nullifier: Debug" + ), + Default(bound = ""), + Eq(bound = "UtxoAccumulatorItem: Eq, UtxoAccumulatorWitness: Eq, Nullifier: Eq"), + Hash( + bound = "UtxoAccumulatorItem: Hash, UtxoAccumulatorWitness: Hash, Nullifier: Hash" + ), + PartialEq( + bound = "UtxoAccumulatorItem: PartialEq, UtxoAccumulatorWitness: PartialEq, Nullifier: PartialEq" + ) +)] +pub struct InitialSyncRequest +where + C: transfer::Configuration + ?Sized, +{ + /// UTXO Data + pub utxo_data: Vec>>, + + /// Membership Proof Data + pub membership_proof_data: Vec>, + + /// Nullifier Count + pub nullifier_count: u128, +} + +impl InitialSyncRequest +where + C: transfer::Configuration, +{ + /// + #[inline] + pub fn from_initial_sync_data(parameters: &Parameters, data: InitialSyncData) -> Self + where + C: Configuration, + { + Self { + utxo_data: C::UtxoAccumulator::sort_items( + data.utxo_data + .iter() + .map(|utxo| functions::item_hash::(parameters, utxo)) + .collect(), + ), + membership_proof_data: data.membership_proof_data, + nullifier_count: data.nullifier_count, + } + } + + /// + #[inline] + pub fn extend_with_data(&mut self, parameters: &Parameters, data: InitialSyncData) + where + C: Configuration, + { + let InitialSyncData { + utxo_data, + membership_proof_data, + nullifier_count, + } = data; + let sorted_utxo_data = C::UtxoAccumulator::sort_items( + utxo_data + .iter() + .map(|utxo| functions::item_hash::(parameters, utxo)) + .collect(), + ); + for (old_vector, new_vector) in self.utxo_data.iter_mut().zip(sorted_utxo_data.into_iter()) + { + old_vector.extend(new_vector) + } + self.membership_proof_data = membership_proof_data; + self.nullifier_count = nullifier_count; + } +} + /// Signer Synchronization Data #[cfg_attr( feature = "serde", @@ -684,6 +787,9 @@ where /// Updates `self` by viewing a new `accumulator`. fn update_from_utxo_accumulator(&mut self, accumulator: &Self::UtxoAccumulator); + /// + fn update_from_utxo_count(&mut self, utxo_count: Vec); + /// Computes a best-effort [`Checkpoint`] from the current `accumulator` state. #[inline] fn from_utxo_accumulator(accumulator: &Self::UtxoAccumulator) -> Self { @@ -1137,10 +1243,9 @@ where #[inline] pub fn initial_sync( &mut self, - request: InitialSyncData, + request: InitialSyncRequest, ) -> Result, SyncError> { functions::intial_sync( - &self.parameters, &mut self.state.assets, &mut self.state.checkpoint, &mut self.state.utxo_accumulator, @@ -1275,6 +1380,12 @@ where } false } + + /// + #[inline] + pub fn transfer_parameters(&self) -> Parameters { + self.parameters.parameters.clone() + } } impl Connection for Signer @@ -1296,7 +1407,7 @@ where #[inline] fn initial_sync( &mut self, - request: InitialSyncData, + request: InitialSyncRequest, ) -> LocalBoxFutureResult, Self::Error> { Box::pin(async move { Ok(self.initial_sync(request)) }) } @@ -1340,6 +1451,11 @@ where { Box::pin(async move { Ok(self.sign_with_transaction_data(request.transaction)) }) } + + #[inline] + fn transfer_parameters(&mut self) -> LocalBoxFutureResult, Self::Error> { + Box::pin(async move { Ok(Signer::transfer_parameters(self)) }) + } } /// Storage State diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index bf92913c4..10409fd2d 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -990,7 +990,7 @@ impl Config { mut event_subscriber: ES, ) -> Result> where - C: Configuration, + C: signer::Configuration, C::AssetValue: AddAssign + SampleUniform, for<'v> &'v C::AssetValue: CheckedSub, L: Ledger @@ -1001,6 +1001,7 @@ impl Config { >, Error: Debug, S: signer::Connection>>::Checkpoint>, + S::Checkpoint: signer::Checkpoint, S::Error: Debug, B: BalanceState, R: CryptoRng + RngCore, diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index d08348392..3f883fc43 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -205,15 +205,16 @@ pub trait OptimizedAccumulator: Accumulator { /// From Items and Witnesses pub trait FromItemsAndWitnesses: Accumulator { - /// Number of Proofs - const NUMBER_OF_PROOFS: usize; - /// Builds a new [`Self`] from `items` and `proofs`. fn from_items_and_witnesses( model: &Self::Model, - items: Vec, + items: Vec>, witnesses: Vec, ) -> Self; + + /// Sorts `items`. + // TODO: move this to the model. + fn sort_items(items: Vec) -> Vec>; } /// Accumulator Membership Proof diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 1927ea48a..c69f5e8e3 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -539,22 +539,21 @@ where #[inline] pub fn from_leaves_and_paths_unchecked( parameters: &Parameters, - leaves: Vec>, + leaves: Vec>>, paths: Vec>, ) -> Self { - TreeArray::new(BoxArray::from_iter(paths.into_iter().enumerate().map( - |(tree_index, path)| { - Partial::from_leaves_and_path_unchecked( - parameters, - leaves - .iter() - .filter(|leaf| C::tree_index(leaf).into() == tree_index) - .map(|leaf| parameters.digest(leaf)) - .collect(), - path, - ) - }, - ))) + TreeArray::new(BoxArray::from_iter( + leaves + .into_iter() + .zip(paths.into_iter()) + .map(|(leaves, path)| { + Partial::from_leaves_and_path_unchecked( + parameters, + leaves.iter().map(|leaf| parameters.digest(leaf)).collect(), + path, + ) + }), + )) } } @@ -571,22 +570,21 @@ where #[inline] pub fn from_leaves_and_paths_unchecked( parameters: &Parameters, - leaves: Vec>, + leaves: Vec>>, paths: Vec>, ) -> Self { - TreeArray::new(BoxArray::from_iter(paths.into_iter().enumerate().map( - |(tree_index, path)| { - ForkedTree::from_leaves_and_path_unchecked( - parameters, - leaves - .iter() - .filter(|leaf| C::tree_index(leaf).into() == tree_index) - .map(|leaf| parameters.digest(leaf)) - .collect(), - path, - ) - }, - ))) + TreeArray::new(BoxArray::from_iter( + leaves + .into_iter() + .zip(paths.into_iter()) + .map(|(leaves, path)| { + ForkedTree::from_leaves_and_path_unchecked( + parameters, + leaves.iter().map(|leaf| parameters.digest(leaf)).collect(), + path, + ) + }), + )) } } @@ -598,20 +596,32 @@ where LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { - const NUMBER_OF_PROOFS: usize = N; - #[inline] fn from_items_and_witnesses( model: &Self::Model, - items: Vec, + items: Vec>, witnesses: Vec, ) -> Self { assert_eq!(witnesses.len(), N); + assert_eq!(items.len(), N); Self::from_forest( TreeArray::, N>::from_leaves_and_paths_unchecked(model, items, witnesses), model.clone(), ) } + + #[inline] + fn sort_items(items: Vec) -> Vec> { + let mut result = Vec::>::default(); + result.resize_with(N, Default::default); + + for item in items { + let tree_index = C::tree_index(&item).into(); + result[tree_index].push(item); + } + + result + } } impl FromItemsAndWitnesses @@ -623,12 +633,10 @@ where LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { - const NUMBER_OF_PROOFS: usize = N; - #[inline] fn from_items_and_witnesses( model: &Self::Model, - items: Vec, + items: Vec>, witnesses: Vec, ) -> Self { assert_eq!(witnesses.len(), N); @@ -639,6 +647,19 @@ where model.clone(), ) } + + #[inline] + fn sort_items(items: Vec) -> Vec> { + let mut result = Vec::>::default(); + result.resize_with(N, Default::default); + + for item in items { + let tree_index = C::tree_index(&item).into(); + result[tree_index].push(item); + } + + result + } } impl AsRef<[T; N]> for TreeArray diff --git a/manta-pay/src/signer/base.rs b/manta-pay/src/signer/base.rs index ab0635a22..50668c90c 100644 --- a/manta-pay/src/signer/base.rs +++ b/manta-pay/src/signer/base.rs @@ -112,6 +112,11 @@ impl signer::Checkpoint for Checkpoint { .collect(); } + #[inline] + fn update_from_utxo_count(&mut self, utxo_count: Vec) { + self.receiver_index = manta_util::Array(utxo_count.try_into().expect("Wrong size")); + } + /// Prunes the `data` by comparing `origin` and `signer_checkpoint` and checks if updating the /// `origin` checkpoint by viewing `data` would exceed the current `signer_checkpoint`. If not, /// then we can prune all the data. Otherwise, we take each entry in `data` and remove by shard diff --git a/manta-pay/src/signer/client/http.rs b/manta-pay/src/signer/client/http.rs index 20448ed48..fa5d0668c 100644 --- a/manta-pay/src/signer/client/http.rs +++ b/manta-pay/src/signer/client/http.rs @@ -17,12 +17,12 @@ //! Signer HTTP Client Implementation use crate::{ - config::{utxo::Address, Config}, + config::{utxo::Address, Config, Parameters}, signer::{ client::network::{Message, Network}, - AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, InitialSyncData, - SignError, SignRequest, SignResponse, SignWithTransactionDataResult, SyncError, - SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, + AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, + InitialSyncRequest, SignError, SignRequest, SignResponse, SignWithTransactionDataResult, + SyncError, SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, }, }; use alloc::boxed::Box; @@ -105,7 +105,7 @@ impl signer::Connection for Client { #[inline] fn initial_sync( &mut self, - request: InitialSyncData, + request: InitialSyncRequest, ) -> LocalBoxFutureResult, Self::Error> { Box::pin(self.post_request("initial_sync", request)) } @@ -146,4 +146,9 @@ impl signer::Connection for Client { ) -> LocalBoxFutureResult { Box::pin(self.post_request("sign_with_transaction_data", request)) } + + #[inline] + fn transfer_parameters(&mut self) -> LocalBoxFutureResult { + Box::pin(self.post_request("transfer_parameters", GetRequest::Get)) + } } diff --git a/manta-pay/src/signer/client/websocket.rs b/manta-pay/src/signer/client/websocket.rs index 376869a5f..6c7811f59 100644 --- a/manta-pay/src/signer/client/websocket.rs +++ b/manta-pay/src/signer/client/websocket.rs @@ -19,11 +19,11 @@ // TODO: Make this code work on WASM and non-WASM by choosing the correct dependency library. use crate::{ - config::{utxo::Address, Config}, + config::{utxo::Address, Config, Parameters}, signer::{ - AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, InitialSyncData, - SignError, SignRequest, SignResponse, SignWithTransactionDataResult, SyncError, - SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, + AssetMetadata, Checkpoint, GetRequest, IdentityRequest, IdentityResponse, + InitialSyncRequest, SignError, SignRequest, SignResponse, SignWithTransactionDataResult, + SyncError, SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, }, }; use alloc::boxed::Box; @@ -143,7 +143,7 @@ impl signer::Connection for Client { #[inline] fn initial_sync( &mut self, - request: InitialSyncData, + request: InitialSyncRequest, ) -> LocalBoxFutureResult, Self::Error> { Box::pin(self.send("initial_sync", request)) } @@ -184,4 +184,9 @@ impl signer::Connection for Client { ) -> LocalBoxFutureResult { Box::pin(self.send("sign_with_transaction_data", request)) } + + #[inline] + fn transfer_parameters(&mut self) -> LocalBoxFutureResult { + Box::pin(self.send("transfer_parameters", GetRequest::Get)) + } } diff --git a/manta-pay/src/signer/mod.rs b/manta-pay/src/signer/mod.rs index 71808ebde..2575e30a7 100644 --- a/manta-pay/src/signer/mod.rs +++ b/manta-pay/src/signer/mod.rs @@ -43,6 +43,9 @@ pub type SyncRequest = signer::SyncRequest; /// Initial Synchronization Data pub type InitialSyncData = signer::InitialSyncData; +/// Initial Synchronization Request +pub type InitialSyncRequest = signer::InitialSyncRequest; + /// Synchronization Response pub type SyncResponse = signer::SyncResponse; diff --git a/manta-pay/src/simulation/mod.rs b/manta-pay/src/simulation/mod.rs index fffe65bc9..47dd5d6dd 100644 --- a/manta-pay/src/simulation/mod.rs +++ b/manta-pay/src/simulation/mod.rs @@ -33,7 +33,7 @@ use manta_accounting::{ asset::AssetList, key::AccountTable, wallet::{ - self, + self, signer, signer::SyncData, test::{self, PublicBalanceOracle}, Error, @@ -173,6 +173,7 @@ impl Simulation { Config, Checkpoint = >>::Checkpoint, >, + S::Checkpoint: signer::Checkpoint, S::Error: Debug, GL: FnMut(usize) -> L, GS: FnMut(usize) -> S, From 1d83f3c35a1781d0e93b6addc63c1f591dc0d01a Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 20 Mar 2023 19:31:29 +0100 Subject: [PATCH 18/58] export type CurrentPath --- manta-pay/src/config/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manta-pay/src/config/mod.rs b/manta-pay/src/config/mod.rs index ae6866c9e..790a7cd98 100644 --- a/manta-pay/src/config/mod.rs +++ b/manta-pay/src/config/mod.rs @@ -25,6 +25,7 @@ use manta_crypto::arkworks::{ self, constraints::EdwardsVar as Bn254_EdwardsVar, EdwardsProjective as Bn254_Edwards, }, groth16, + merkle_tree::path::CurrentPath, }; #[cfg(feature = "bs58")] @@ -149,6 +150,9 @@ pub type AssetValue = transfer::AssetValue; /// Asset Type pub type Asset = transfer::Asset; +/// Current Path Type +pub type CurrentPath = CurrentPath; + /// Unspent Transaction Output Type pub type Utxo = transfer::Utxo; From 976ae741bafd143baa520bcc930221383ba9dcbf Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 20 Mar 2023 19:43:52 +0100 Subject: [PATCH 19/58] small error --- manta-pay/src/config/mod.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/manta-pay/src/config/mod.rs b/manta-pay/src/config/mod.rs index 790a7cd98..d61f7a51e 100644 --- a/manta-pay/src/config/mod.rs +++ b/manta-pay/src/config/mod.rs @@ -17,15 +17,17 @@ //! Manta-Pay Configuration use manta_accounting::transfer; -use manta_crypto::arkworks::{ - algebra::{self, ScalarVar}, - bn254::{self, Bn254}, - constraint::{FpVar, R1CS}, - ed_on_bn254::{ - self, constraints::EdwardsVar as Bn254_EdwardsVar, EdwardsProjective as Bn254_Edwards, +use manta_crypto::{ + arkworks::{ + algebra::{self, ScalarVar}, + bn254::{self, Bn254}, + constraint::{FpVar, R1CS}, + ed_on_bn254::{ + self, constraints::EdwardsVar as Bn254_EdwardsVar, EdwardsProjective as Bn254_Edwards, + }, + groth16, }, - groth16, - merkle_tree::path::CurrentPath, + merkle_tree::path, }; #[cfg(feature = "bs58")] @@ -151,7 +153,7 @@ pub type AssetValue = transfer::AssetValue; pub type Asset = transfer::Asset; /// Current Path Type -pub type CurrentPath = CurrentPath; +pub type CurrentPath = path::CurrentPath; /// Unspent Transaction Output Type pub type Utxo = transfer::Utxo; From e16c651596470e8c0a4ae4df8d586bfc784e5ace Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 21 Mar 2023 20:34:42 +0100 Subject: [PATCH 20/58] debugging --- manta-accounting/src/wallet/mod.rs | 4 +++- manta-accounting/src/wallet/test/mod.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 97e7a37e4..bf0d9e7d6 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -31,7 +31,7 @@ use crate::{ asset::AssetList, transfer::{ canonical::{Transaction, TransactionKind}, - Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorModel, + Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorModel, UtxoAccumulatorItem, }, wallet::{ balance::{BTreeMapBalanceState, BalanceState}, @@ -274,6 +274,7 @@ where L: ledger::Read, Checkpoint = S::Checkpoint>, C: signer::Configuration, S::Checkpoint: signer::Checkpoint, + UtxoAccumulatorItem: Debug, { let mut is_continue = true; let mut checkpoint = self.checkpoint.clone(); @@ -296,6 +297,7 @@ where .map_err(Error::SignerConnectionError)?, data, ); + panic!("Utxo data: {:?}", request.utxo_data); checkpoint .update_from_utxo_count(request.utxo_data.iter().map(|utxos| utxos.len()).collect()) } diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index 10409fd2d..be4aaec3f 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -21,7 +21,7 @@ use crate::{ asset::AssetList, - transfer::{canonical::Transaction, Address, Asset, Configuration, TransferPost}, + transfer::{canonical::Transaction, Address, Asset, Configuration, TransferPost, UtxoAccumulatorItem}, wallet::{ ledger, signer::{self, InitialSyncData, SyncData}, @@ -1012,6 +1012,7 @@ impl Config { ES: Copy + FnMut(&sim::Event>>) -> ESFut, ESFut: Future, Address: Clone + Eq + Hash, + UtxoAccumulatorItem: Debug, { let action_distribution = ActionDistribution::try_from(self.action_distribution) .expect("Unable to sample from action distribution."); From 0c1506dc32c1d35ff587a33eeab0670828fb6b03 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 21 Mar 2023 20:43:07 +0100 Subject: [PATCH 21/58] debug --- manta-accounting/src/wallet/mod.rs | 1 - manta-accounting/src/wallet/signer/mod.rs | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index bf0d9e7d6..2dd87d72d 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -297,7 +297,6 @@ where .map_err(Error::SignerConnectionError)?, data, ); - panic!("Utxo data: {:?}", request.utxo_data); checkpoint .update_from_utxo_count(request.utxo_data.iter().map(|utxos| utxos.len()).collect()) } diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 1d5c8325b..ecf75fa65 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -244,6 +244,7 @@ where pub fn extend_with_data(&mut self, parameters: &Parameters, data: InitialSyncData) where C: Configuration, + UtxoAccumulatorItem: Debug, { let InitialSyncData { utxo_data, @@ -256,6 +257,7 @@ where .map(|utxo| functions::item_hash::(parameters, utxo)) .collect(), ); + panic!("Sorted utxo data: {:?}", sorted_utxo_data); for (old_vector, new_vector) in self.utxo_data.iter_mut().zip(sorted_utxo_data.into_iter()) { old_vector.extend(new_vector) From bcd0aff73d174cb811d42d93c818349e14d092b3 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 21 Mar 2023 21:22:02 +0100 Subject: [PATCH 22/58] debugging --- manta-accounting/src/wallet/signer/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index ecf75fa65..5f3d924ec 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -257,11 +257,11 @@ where .map(|utxo| functions::item_hash::(parameters, utxo)) .collect(), ); - panic!("Sorted utxo data: {:?}", sorted_utxo_data); for (old_vector, new_vector) in self.utxo_data.iter_mut().zip(sorted_utxo_data.into_iter()) { old_vector.extend(new_vector) } + panic!("Utxo data: {:?}", self.utxo_data); self.membership_proof_data = membership_proof_data; self.nullifier_count = nullifier_count; } From 69a088bb12af27ba07de373fb0655d033252ab36 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 21 Mar 2023 21:32:49 +0100 Subject: [PATCH 23/58] changed default impl --- manta-accounting/src/wallet/mod.rs | 3 ++- manta-accounting/src/wallet/signer/mod.rs | 20 ++++++++++++++++++-- manta-accounting/src/wallet/test/mod.rs | 4 +++- manta-crypto/src/accumulator.rs | 3 +++ manta-crypto/src/merkle_tree/forest.rs | 4 ++++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 2dd87d72d..a313d4391 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -31,7 +31,8 @@ use crate::{ asset::AssetList, transfer::{ canonical::{Transaction, TransactionKind}, - Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorModel, UtxoAccumulatorItem, + Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorItem, + UtxoAccumulatorModel, }, wallet::{ balance::{BTreeMapBalanceState, BalanceState}, diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 5f3d924ec..e9a9e25a2 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -194,7 +194,6 @@ where Debug( bound = "UtxoAccumulatorItem: Debug, UtxoAccumulatorWitness: Debug, Nullifier: Debug" ), - Default(bound = ""), Eq(bound = "UtxoAccumulatorItem: Eq, UtxoAccumulatorWitness: Eq, Nullifier: Eq"), Hash( bound = "UtxoAccumulatorItem: Hash, UtxoAccumulatorWitness: Hash, Nullifier: Hash" @@ -261,12 +260,29 @@ where { old_vector.extend(new_vector) } - panic!("Utxo data: {:?}", self.utxo_data); self.membership_proof_data = membership_proof_data; self.nullifier_count = nullifier_count; } } +impl Default for InitialSyncRequest +where + C: Configuration, +{ + #[inline] + fn default() -> Self { + let mut utxo_data = Vec::>::default(); + let mut membership_proof_data = Vec::new(); + utxo_data.resize_with(C::UtxoAccumulator::NUMBER_OF_PROOFS, Default::default); + membership_proof_data.resize_with(C::UtxoAccumulator::NUMBER_OF_PROOFS, Default::default); + Self { + utxo_data, + membership_proof_data, + nullifier_count: Default::default(), + } + } +} + /// Signer Synchronization Data #[cfg_attr( feature = "serde", diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index be4aaec3f..a8cfa97ab 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -21,7 +21,9 @@ use crate::{ asset::AssetList, - transfer::{canonical::Transaction, Address, Asset, Configuration, TransferPost, UtxoAccumulatorItem}, + transfer::{ + canonical::Transaction, Address, Asset, Configuration, TransferPost, UtxoAccumulatorItem, + }, wallet::{ ledger, signer::{self, InitialSyncData, SyncData}, diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 3f883fc43..5d2f3feda 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -205,6 +205,9 @@ pub trait OptimizedAccumulator: Accumulator { /// From Items and Witnesses pub trait FromItemsAndWitnesses: Accumulator { + /// + const NUMBER_OF_PROOFS: usize; + /// Builds a new [`Self`] from `items` and `proofs`. fn from_items_and_witnesses( model: &Self::Model, diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index c69f5e8e3..4e8df6eff 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -596,6 +596,8 @@ where LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { + const NUMBER_OF_PROOFS: usize = N; + #[inline] fn from_items_and_witnesses( model: &Self::Model, @@ -633,6 +635,8 @@ where LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { + const NUMBER_OF_PROOFS: usize = N; + #[inline] fn from_items_and_witnesses( model: &Self::Model, From 087e7b65ca35bd21f4ba584c313c9bd091acc478 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 14:32:27 +0100 Subject: [PATCH 24/58] first round of corrections --- manta-accounting/src/transfer/mod.rs | 4 +- manta-accounting/src/wallet/mod.rs | 26 ++- .../src/wallet/signer/functions.rs | 12 +- manta-accounting/src/wallet/signer/mod.rs | 30 ++- manta-crypto/src/accumulator.rs | 2 +- manta-crypto/src/merkle_tree/forest.rs | 212 +++++++----------- manta-crypto/src/merkle_tree/inner_tree.rs | 2 +- manta-pay/src/signer/base.rs | 7 +- 8 files changed, 136 insertions(+), 159 deletions(-) diff --git a/manta-accounting/src/transfer/mod.rs b/manta-accounting/src/transfer/mod.rs index 5e6d89da1..ccd9980d4 100644 --- a/manta-accounting/src/transfer/mod.rs +++ b/manta-accounting/src/transfer/mod.rs @@ -148,11 +148,11 @@ pub trait Configuration { type UtxoAccumulatorItemHash: ItemHashFunction, Item = UtxoAccumulatorItem>; /// Parameters Type - type Parameters: auth::DeriveContext + type Parameters: Clone + + auth::DeriveContext + auth::ProveAuthorization + auth::VerifyAuthorization + auth::DeriveSigningKey - + Clone + for<'a> auth::Sign> + for<'a> auth::VerifySignature> + utxo::AssetType> diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index a313d4391..b1cf14c2f 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -39,8 +39,9 @@ use crate::{ ledger::ReadResponse, signer::{ BalanceUpdate, Checkpoint, IdentityRequest, IdentityResponse, InitialSyncData, - SignError, SignRequest, SignResponse, SignWithTransactionDataResponse, SyncData, - SyncError, SyncRequest, SyncResponse, TransactionDataRequest, TransactionDataResponse, + InitialSyncRequest, SignError, SignRequest, SignResponse, + SignWithTransactionDataResponse, SyncData, SyncError, SyncRequest, SyncResponse, + TransactionDataRequest, TransactionDataResponse, }, }, }; @@ -51,8 +52,6 @@ use manta_util::ops::ControlFlow; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; -use self::signer::InitialSyncRequest; - pub mod balance; pub mod ledger; pub mod signer; @@ -268,7 +267,24 @@ where Ok(()) } + /// Pulls data from the ledger, synchronizing the wallet and balance state. This method + /// builds a [`InitialSyncRequest`] by continuously calling [`read`](ledger::Read::read) + /// until all the ledger data has arrived. Once the request is built, it executes + /// synchronizes the signer against it. + /// + /// # Implementation Note + /// + /// Using this method to synchronize a signer will make it impossibile to spend any + /// [`Utxo`](crate::transfer::Utxo)s already on the ledger at the time of synchronization. + /// Therefore, this method should only be used for the initial synchronization of a + /// new signer. + /// + /// # Failure Conditions /// + /// This method returns an element of type [`Error`] on failure, which can result from any + /// number of synchronization issues between the wallet, the ledger, and the signer. See the + /// [`InconsistencyError`] type for more information on the kinds of errors that can occur and + /// how to resolve them. #[inline] pub async fn initial_sync(&mut self) -> Result<(), Error> where @@ -390,7 +406,7 @@ where } } - /// + /// Performs an initial synchronization with the signer against the given `request`. #[inline] async fn signer_initial_sync( &mut self, diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 3c60b8765..9959612cf 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -35,9 +35,9 @@ use crate::{ UtxoAccumulatorItem, UtxoAccumulatorModel, UtxoAccumulatorWitness, }, wallet::signer::{ - AccountTable, BalanceUpdate, Checkpoint, Configuration, SignError, SignResponse, - SignWithTransactionDataResponse, SignWithTransactionDataResult, SignerParameters, SyncData, - SyncError, SyncRequest, SyncResponse, + AccountTable, BalanceUpdate, Checkpoint, Configuration, InitialSyncRequest, SignError, + SignResponse, SignWithTransactionDataResponse, SignWithTransactionDataResult, + SignerParameters, SyncData, SyncError, SyncRequest, SyncResponse, }, }; use alloc::{vec, vec::Vec}; @@ -50,8 +50,6 @@ use manta_util::{ vec::VecExt, }; -use super::InitialSyncRequest; - /// Returns the default account for `accounts`. #[inline] pub fn default_account(accounts: &AccountTable) -> Account @@ -965,8 +963,8 @@ where Ok(response) } -/// Updates the internal ledger state, returning the new asset distribution. -#[allow(clippy::too_many_arguments)] +/// Updates the internal ledger state from `utxos`, `membership_proof_data` +/// and `nullifier_count`. #[inline] fn initial_sync_with( assets: &mut C::AssetMap, diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index e9a9e25a2..dcc0da1ca 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -79,7 +79,14 @@ where request: SyncRequest, ) -> LocalBoxFutureResult, Self::Error>; + /// Performs the initial synchronization of a new signer with the ledger data. /// + /// # Implementation Note + /// + /// Using this method to synchronize a signer will make it impossibile to spend any + /// [`Utxo`](crate::transfer::Utxo)s already on the ledger at the time of synchronization. + /// Therefore, this method should only be used for the initial synchronization of a + /// new signer. fn initial_sync( &mut self, request: InitialSyncRequest, @@ -116,7 +123,7 @@ where where TransferPost: Clone; - /// + /// Returns the transfer [`Parameters`] corresponding to `self`. fn transfer_parameters(&mut self) -> LocalBoxFutureResult, Self::Error>; } @@ -220,7 +227,7 @@ impl InitialSyncRequest where C: transfer::Configuration, { - /// + /// Builds a new [`InitialSyncRequest`] from `parameters` and `data`. #[inline] pub fn from_initial_sync_data(parameters: &Parameters, data: InitialSyncData) -> Self where @@ -238,7 +245,7 @@ where } } - /// + /// Extends `self` with `parameters` and `data`. #[inline] pub fn extend_with_data(&mut self, parameters: &Parameters, data: InitialSyncData) where @@ -805,7 +812,7 @@ where /// Updates `self` by viewing a new `accumulator`. fn update_from_utxo_accumulator(&mut self, accumulator: &Self::UtxoAccumulator); - /// + /// Updates `self` by viewing `utxo_count`-many [`Utxo`]s. fn update_from_utxo_count(&mut self, utxo_count: Vec); /// Computes a best-effort [`Checkpoint`] from the current `accumulator` state. @@ -1257,7 +1264,14 @@ where ) } + /// Performs the initial synchronization of a new signer with the ledger data. + /// + /// # Implementation Note /// + /// Using this method to synchronize a signer will make it impossibile to spend any + /// [`Utxo`](crate::transfer::Utxo)s already on the ledger at the time of synchronization. + /// Therefore, this method should only be used for the initial synchronization of a + /// new signer. #[inline] pub fn initial_sync( &mut self, @@ -1399,10 +1413,10 @@ where false } - /// + /// Returns the transfer [`Parameters`] corresponding to `self`. #[inline] - pub fn transfer_parameters(&self) -> Parameters { - self.parameters.parameters.clone() + pub fn transfer_parameters(&self) -> &Parameters { + &self.parameters.parameters } } @@ -1472,7 +1486,7 @@ where #[inline] fn transfer_parameters(&mut self) -> LocalBoxFutureResult, Self::Error> { - Box::pin(async move { Ok(Signer::transfer_parameters(self)) }) + Box::pin(async move { Ok(Signer::transfer_parameters(self).clone()) }) } } diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 5d2f3feda..1f0bd9bec 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -205,7 +205,7 @@ pub trait OptimizedAccumulator: Accumulator { /// From Items and Witnesses pub trait FromItemsAndWitnesses: Accumulator { - /// + /// Number of Proofs const NUMBER_OF_PROOFS: usize; /// Builds a new [`Self`] from `items` and `proofs`. diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 4e8df6eff..e8675b4d4 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -526,145 +526,89 @@ where } } -impl TreeArray, N> -where - C: Configuration + ?Sized, - C::Index: FixedIndex, - LeafDigest: Clone + Default, - InnerDigest: Clone + Default + PartialEq, -{ - /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that - /// the `paths` are consistent with the leaves and that they are - /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. - #[inline] - pub fn from_leaves_and_paths_unchecked( - parameters: &Parameters, - leaves: Vec>>, - paths: Vec>, - ) -> Self { - TreeArray::new(BoxArray::from_iter( - leaves - .into_iter() - .zip(paths.into_iter()) - .map(|(leaves, path)| { - Partial::from_leaves_and_path_unchecked( - parameters, - leaves.iter().map(|leaf| parameters.digest(leaf)).collect(), - path, - ) - }), - )) - } -} - -impl TreeArray>, N> -where - C: Configuration + ?Sized, - C::Index: FixedIndex, - LeafDigest: Clone + Default, - InnerDigest: Clone + Default + PartialEq, -{ - /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that - /// the `paths` are consistent with the leaves and that they are - /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. - #[inline] - pub fn from_leaves_and_paths_unchecked( - parameters: &Parameters, - leaves: Vec>>, - paths: Vec>, - ) -> Self { - TreeArray::new(BoxArray::from_iter( - leaves - .into_iter() - .zip(paths.into_iter()) - .map(|(leaves, path)| { - ForkedTree::from_leaves_and_path_unchecked( - parameters, - leaves.iter().map(|leaf| parameters.digest(leaf)).collect(), - path, - ) - }), - )) - } -} - -impl FromItemsAndWitnesses for TreeArrayMerkleForest, N> -where - C: Configuration + ?Sized, - C::Index: FixedIndex, - Parameters: Clone, - LeafDigest: Clone + Default + PartialEq, - InnerDigest: Clone + Default + PartialEq, -{ - const NUMBER_OF_PROOFS: usize = N; - - #[inline] - fn from_items_and_witnesses( - model: &Self::Model, - items: Vec>, - witnesses: Vec, - ) -> Self { - assert_eq!(witnesses.len(), N); - assert_eq!(items.len(), N); - Self::from_forest( - TreeArray::, N>::from_leaves_and_paths_unchecked(model, items, witnesses), - model.clone(), - ) - } - - #[inline] - fn sort_items(items: Vec) -> Vec> { - let mut result = Vec::>::default(); - result.resize_with(N, Default::default); - - for item in items { - let tree_index = C::tree_index(&item).into(); - result[tree_index].push(item); +macro_rules! impl_from_items_and_witnesses { + ($forest:ty, $tree_array:ty, $tree_variant:ty) => { + impl $tree_array + where + C: Configuration + ?Sized, + C::Index: FixedIndex, + LeafDigest: Clone + Default, + InnerDigest: Clone + Default + PartialEq, + { + /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that + /// the `paths` are consistent with the leaves and that they are + /// [`CurrentPath`](crate::merkle_tree::path::CurrentPath)s. + #[inline] + pub fn from_leaves_and_paths_unchecked( + parameters: &Parameters, + leaves: Vec>>, + paths: Vec>, + ) -> Self { + <$tree_array>::new(BoxArray::from_iter( + leaves + .into_iter() + .zip(paths.into_iter()) + .map(|(leaves, path)| { + <$tree_variant>::from_leaves_and_path_unchecked( + parameters, + leaves.iter().map(|leaf| parameters.digest(leaf)).collect(), + path, + ) + }), + )) + } } - result - } + impl FromItemsAndWitnesses for $forest + where + C: Configuration + ?Sized, + C::Index: FixedIndex, + Parameters: Clone, + LeafDigest: Clone + Default + PartialEq, + InnerDigest: Clone + Default + PartialEq, + { + const NUMBER_OF_PROOFS: usize = N; + + #[inline] + fn from_items_and_witnesses( + model: &Self::Model, + items: Vec>, + witnesses: Vec, + ) -> Self { + assert_eq!(witnesses.len(), N); + Self::from_forest( + <$tree_array>::from_leaves_and_paths_unchecked(model, items, witnesses), + model.clone(), + ) + } + + #[inline] + fn sort_items(items: Vec) -> Vec> { + let mut result = Vec::>::default(); + result.resize_with(N, Default::default); + + for item in items { + let tree_index = C::tree_index(&item).into(); + result[tree_index].push(item); + } + + result + } + } + }; } -impl FromItemsAndWitnesses - for TreeArrayMerkleForest>, N> -where - C: Configuration + ?Sized, - C::Index: FixedIndex, - Parameters: Clone, - LeafDigest: Clone + Default + PartialEq, - InnerDigest: Clone + Default + PartialEq, -{ - const NUMBER_OF_PROOFS: usize = N; - - #[inline] - fn from_items_and_witnesses( - model: &Self::Model, - items: Vec>, - witnesses: Vec, - ) -> Self { - assert_eq!(witnesses.len(), N); - Self::from_forest( - TreeArray::>, N>::from_leaves_and_paths_unchecked( - model, items, witnesses, - ), - model.clone(), - ) - } - - #[inline] - fn sort_items(items: Vec) -> Vec> { - let mut result = Vec::>::default(); - result.resize_with(N, Default::default); - - for item in items { - let tree_index = C::tree_index(&item).into(); - result[tree_index].push(item); - } +impl_from_items_and_witnesses!( + TreeArrayMerkleForest, N>, + TreeArray, N>, + Partial +); - result - } -} +impl_from_items_and_witnesses!( + TreeArrayMerkleForest>, N>, + TreeArray>, N>, + ForkedTree> +); impl AsRef<[T; N]> for TreeArray where diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index 42eca5012..7e769dcaf 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -775,7 +775,7 @@ where self.inner_tree.path(leaf_index) } - /// + /// Sets the starting leaf index in `self` to `default`. #[inline] pub fn reset_starting_leaf_index(&mut self, default: Node) { self.starting_leaf_index = default; diff --git a/manta-pay/src/signer/base.rs b/manta-pay/src/signer/base.rs index 50668c90c..bcba9b7e9 100644 --- a/manta-pay/src/signer/base.rs +++ b/manta-pay/src/signer/base.rs @@ -114,7 +114,12 @@ impl signer::Checkpoint for Checkpoint { #[inline] fn update_from_utxo_count(&mut self, utxo_count: Vec) { - self.receiver_index = manta_util::Array(utxo_count.try_into().expect("Wrong size")); + self.receiver_index = manta_util::Array(utxo_count.try_into().unwrap_or_else(|_| { + panic!( + "Utxo count must have {} elements", + MerkleTreeConfiguration::FOREST_WIDTH + ) + })) } /// Prunes the `data` by comparing `origin` and `signer_checkpoint` and checks if updating the From 09b5969e255a6a18ccb735e7255b9afb4d4720f1 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 14:38:48 +0100 Subject: [PATCH 25/58] debug removed --- manta-accounting/src/wallet/mod.rs | 1 - manta-accounting/src/wallet/signer/mod.rs | 1 - manta-accounting/src/wallet/test/mod.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index b1cf14c2f..64e1d05d2 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -291,7 +291,6 @@ where L: ledger::Read, Checkpoint = S::Checkpoint>, C: signer::Configuration, S::Checkpoint: signer::Checkpoint, - UtxoAccumulatorItem: Debug, { let mut is_continue = true; let mut checkpoint = self.checkpoint.clone(); diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index dcc0da1ca..c54f00e95 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -250,7 +250,6 @@ where pub fn extend_with_data(&mut self, parameters: &Parameters, data: InitialSyncData) where C: Configuration, - UtxoAccumulatorItem: Debug, { let InitialSyncData { utxo_data, diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index a8cfa97ab..6caf05bd7 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -1014,7 +1014,6 @@ impl Config { ES: Copy + FnMut(&sim::Event>>) -> ESFut, ESFut: Future, Address: Clone + Eq + Hash, - UtxoAccumulatorItem: Debug, { let action_distribution = ActionDistribution::try_from(self.action_distribution) .expect("Unable to sample from action distribution."); From aa4b43be934c7f50fa3fb1d54631eec65b9cc511 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 14:39:26 +0100 Subject: [PATCH 26/58] unused imports --- manta-accounting/src/wallet/mod.rs | 2 +- manta-accounting/src/wallet/test/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 64e1d05d2..5f40f8b16 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -31,7 +31,7 @@ use crate::{ asset::AssetList, transfer::{ canonical::{Transaction, TransactionKind}, - Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorItem, + Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorModel, }, wallet::{ diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index 6caf05bd7..68fec709e 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -22,7 +22,7 @@ use crate::{ asset::AssetList, transfer::{ - canonical::Transaction, Address, Asset, Configuration, TransferPost, UtxoAccumulatorItem, + canonical::Transaction, Address, Asset, Configuration, TransferPost, }, wallet::{ ledger, From 30fe0d8dd2a9f7900499b5e1a56d40073b60c313 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 14:58:30 +0100 Subject: [PATCH 27/58] simplified initial sync. added reset to initial sync --- manta-accounting/src/wallet/mod.rs | 20 ++++++++----------- .../src/wallet/signer/functions.rs | 6 +++++- manta-accounting/src/wallet/test/mod.rs | 4 +--- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 5f40f8b16..8cc79b558 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -31,8 +31,7 @@ use crate::{ asset::AssetList, transfer::{ canonical::{Transaction, TransactionKind}, - Address, Asset, Configuration, IdentifiedAsset, TransferPost, - UtxoAccumulatorModel, + Address, Asset, Configuration, IdentifiedAsset, TransferPost, UtxoAccumulatorModel, }, wallet::{ balance::{BTreeMapBalanceState, BalanceState}, @@ -293,7 +292,7 @@ where S::Checkpoint: signer::Checkpoint, { let mut is_continue = true; - let mut checkpoint = self.checkpoint.clone(); + let mut checkpoint = Default::default(); let mut request = InitialSyncRequest::::default(); while is_continue { let ReadResponse { @@ -433,16 +432,13 @@ where self.checkpoint = checkpoint; Ok(()) } - Err(SyncError::InconsistentSynchronization { checkpoint }) => { - if checkpoint < self.checkpoint { - self.checkpoint = checkpoint; - } - Err(Error::Inconsistency( - InconsistencyError::SignerSynchronization, - )) + Err(SyncError::InconsistentSynchronization { checkpoint: _ }) => { + unreachable!("Initial synchronization always starts at the default checkpoint.") } - Err(SyncError::MissingProofAuthorizationKey) => { - Err(Error::MissingProofAuthorizationKey) + _ => { + unreachable!( + "Proof authorization key is not required for the initial synchronization." + ); } } } diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 9959612cf..431e17ec0 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -944,12 +944,16 @@ pub fn intial_sync( ) -> Result, SyncError> where C: Configuration, + C::AssetMap: Default, + C::Checkpoint: Default, { let InitialSyncRequest { utxo_data, membership_proof_data, nullifier_count, } = request; + *checkpoint = Default::default(); + *assets = Default::default(); let (accumulator, response) = initial_sync_with::( assets, checkpoint, @@ -967,7 +971,7 @@ where /// and `nullifier_count`. #[inline] fn initial_sync_with( - assets: &mut C::AssetMap, + assets: &C::AssetMap, checkpoint: &mut C::Checkpoint, utxo_accumulator_model: &UtxoAccumulatorModel, utxos: Vec>>, diff --git a/manta-accounting/src/wallet/test/mod.rs b/manta-accounting/src/wallet/test/mod.rs index 68fec709e..10409fd2d 100644 --- a/manta-accounting/src/wallet/test/mod.rs +++ b/manta-accounting/src/wallet/test/mod.rs @@ -21,9 +21,7 @@ use crate::{ asset::AssetList, - transfer::{ - canonical::Transaction, Address, Asset, Configuration, TransferPost, - }, + transfer::{canonical::Transaction, Address, Asset, Configuration, TransferPost}, wallet::{ ledger, signer::{self, InitialSyncData, SyncData}, From c2a461043c92f8c139bc91e4dc2d4a18486c6d6c Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 15:30:31 +0100 Subject: [PATCH 28/58] simulation restored --- manta-pay/src/bin/simulation.rs | 2 -- manta-pay/src/simulation/mod.rs | 9 +-------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/manta-pay/src/bin/simulation.rs b/manta-pay/src/bin/simulation.rs index afdb3f916..1d43790a9 100644 --- a/manta-pay/src/bin/simulation.rs +++ b/manta-pay/src/bin/simulation.rs @@ -46,5 +46,3 @@ pub fn main() { .exit(), } } - -// cargo run --release --package manta-pay --all-features --bin simulation 3 10 2 1000 diff --git a/manta-pay/src/simulation/mod.rs b/manta-pay/src/simulation/mod.rs index 47dd5d6dd..dd9d023bf 100644 --- a/manta-pay/src/simulation/mod.rs +++ b/manta-pay/src/simulation/mod.rs @@ -140,14 +140,7 @@ impl Simulation { self.setup(&mut ledger); let ledger = Arc::new(RwLock::new(ledger)); self.run_with( - |i| LedgerConnection::new(account_id_from_u64(i as u64), ledger.clone()), - |_| sample_signer(proving_context, parameters, utxo_accumulator_model, rng), - move |i| account_id_from_u64(i as u64), - ) - .await; - println!("{:?}", ledger.read().await.utxos()); - self.run_with( - |i| LedgerConnection::new(account_id_from_u64(i as u64), ledger.clone()), + move |i| LedgerConnection::new(account_id_from_u64(i as u64), ledger.clone()), move |_| sample_signer(proving_context, parameters, utxo_accumulator_model, rng), move |i| account_id_from_u64(i as u64), ) From c3bd7549900302feb0a5be58b6e574c7d1a106f6 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 17:21:42 +0100 Subject: [PATCH 29/58] testing fixed --- manta-crypto/src/merkle_tree/test/mod.rs | 17 ++++++++--------- manta-crypto/src/merkle_tree/test/partial.rs | 2 +- manta-pay/src/simulation/ledger/mod.rs | 6 ++++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manta-crypto/src/merkle_tree/test/mod.rs b/manta-crypto/src/merkle_tree/test/mod.rs index e01c4d75b..d13720a15 100644 --- a/manta-crypto/src/merkle_tree/test/mod.rs +++ b/manta-crypto/src/merkle_tree/test/mod.rs @@ -294,6 +294,7 @@ pub enum BinaryIndex { } impl From for usize { + #[inline] fn from(value: BinaryIndex) -> Self { match value { BinaryIndex::Zero => 0, @@ -303,11 +304,11 @@ impl From for usize { } impl forest::FixedIndex<2> for BinaryIndex { + #[inline] fn from_index(index: usize) -> Self { - if index % 2 == 0 { - BinaryIndex::Zero - } else { - BinaryIndex::One + match index % 2 { + 0 => BinaryIndex::Zero, + _ => BinaryIndex::One, } } } @@ -317,11 +318,9 @@ impl forest::Configuration for Test { #[inline] fn tree_index(leaf: &Leaf) -> Self::Index { - let parity = leaf % 2; - if parity == 0 { - BinaryIndex::Zero - } else { - BinaryIndex::One + match leaf % 2 { + 0 => BinaryIndex::Zero, + _ => BinaryIndex::One, } } } diff --git a/manta-crypto/src/merkle_tree/test/partial.rs b/manta-crypto/src/merkle_tree/test/partial.rs index cf04639ee..f59f61f2e 100644 --- a/manta-crypto/src/merkle_tree/test/partial.rs +++ b/manta-crypto/src/merkle_tree/test/partial.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with manta-rs. If not, see . -//! Partial Merkle Tree +//! Partial Merkle Tree Tests use crate::{ accumulator::{Accumulator, FromItemsAndWitnesses}, diff --git a/manta-pay/src/simulation/ledger/mod.rs b/manta-pay/src/simulation/ledger/mod.rs index fd30b3c14..b26220f2d 100644 --- a/manta-pay/src/simulation/ledger/mod.rs +++ b/manta-pay/src/simulation/ledger/mod.rs @@ -218,7 +218,9 @@ impl Ledger { true } - /// + /// Pulls the data from the ledger necessary to perform an [`initial_sync`]. + /// + /// [`initial_sync`]: manta_accounting::wallet::signer::Connection::initial_sync #[inline] pub fn initial_read(&self) -> ReadResponse { let mut utxos = Vec::new(); @@ -252,7 +254,7 @@ impl Ledger { } } - /// + /// Returns the [`Utxo`]s in `self`. #[inline] pub fn utxos(&self) -> &HashSet { &self.utxos From c99cf89464da242a22f6ccd0394b521a7fe9d0ff Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 17:24:42 +0100 Subject: [PATCH 30/58] test fixed --- manta-crypto/src/merkle_tree/test/partial.rs | 5 ++++- manta-pay/src/simulation/ledger/mod.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/manta-crypto/src/merkle_tree/test/partial.rs b/manta-crypto/src/merkle_tree/test/partial.rs index f59f61f2e..332d8cc69 100644 --- a/manta-crypto/src/merkle_tree/test/partial.rs +++ b/manta-crypto/src/merkle_tree/test/partial.rs @@ -122,10 +122,13 @@ fn test_from_leaves_and_path_forest() { let path_1 = Path::from(forest.forest.get(BinaryIndex::Zero).current_path()); let path_2 = Path::from(forest.forest.get(BinaryIndex::One).current_path()); let paths = vec![path_1, path_2]; + let items = TreeArrayMerkleForest::<_, ForkedTree<_, Partial>, 2>::sort_items( + insertions.clone(), + ); let partial_forest = TreeArrayMerkleForest::<_, ForkedTree<_, Partial>, 2>::from_items_and_witnesses( ¶meters, - insertions.clone(), + items, paths, ); for leaf in &insertions { diff --git a/manta-pay/src/simulation/ledger/mod.rs b/manta-pay/src/simulation/ledger/mod.rs index b26220f2d..48e4c242c 100644 --- a/manta-pay/src/simulation/ledger/mod.rs +++ b/manta-pay/src/simulation/ledger/mod.rs @@ -219,7 +219,7 @@ impl Ledger { } /// Pulls the data from the ledger necessary to perform an [`initial_sync`]. - /// + /// /// [`initial_sync`]: manta_accounting::wallet::signer::Connection::initial_sync #[inline] pub fn initial_read(&self) -> ReadResponse { From 8bd829ca942a4864fb88fb2c09114a6716cd40a5 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 17:45:31 +0100 Subject: [PATCH 31/58] fixed feature issue --- manta-crypto/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manta-crypto/Cargo.toml b/manta-crypto/Cargo.toml index d8ad63cec..9d523c499 100644 --- a/manta-crypto/Cargo.toml +++ b/manta-crypto/Cargo.toml @@ -94,4 +94,4 @@ rand_chacha = { version = "0.3.1", optional = true, default-features = false } rand_core = { version = "0.6.4", default-features = false } [dev-dependencies] -manta-crypto = { path = ".", default-features = false, features = ["ark-bn254", "ark-ed-on-bn254", "getrandom", "std", "test"] } +manta-crypto = { path = ".", default-features = false, features = ["ark-bn254", "ark-ed-on-bn254", "getrandom", "rand", "std", "test"] } From c87f67b6ed21ec8ce51e229f3de6be232aced0e8 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 17:49:44 +0100 Subject: [PATCH 32/58] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fed7d5ef..af0135153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added +- [\#329](https://github.com/Manta-Network/manta-rs/pull/329) Signer initial synchronization method. - [\#328](https://github.com/Manta-Network/manta-rs/pull/328) Expose reset wallet method. ### Changed From 91901dec1c131582d84c9ae12a9d5959567e80ac Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 22 Mar 2023 17:58:41 +0100 Subject: [PATCH 33/58] signer method in wallet --- manta-accounting/src/wallet/mod.rs | 6 ++++++ manta-pay/src/simulation/mod.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 8cc79b558..9c8909f04 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -145,6 +145,12 @@ where Self::new_unchecked(ledger, Default::default(), signer, Default::default()) } + /// Returns the [`Connection`](signer::Connection). + #[inline] + pub fn signer(&self) -> &S { + &self.signer + } + /// Returns a mutable reference to the [`Connection`](signer::Connection). /// /// # Crypto Safety diff --git a/manta-pay/src/simulation/mod.rs b/manta-pay/src/simulation/mod.rs index dd9d023bf..e4f65a9f7 100644 --- a/manta-pay/src/simulation/mod.rs +++ b/manta-pay/src/simulation/mod.rs @@ -144,7 +144,7 @@ impl Simulation { move |_| sample_signer(proving_context, parameters, utxo_accumulator_model, rng), move |i| account_id_from_u64(i as u64), ) - .await; + .await } /// Runs the simulation with the given ledger connections and signer connections. From 090b67ef5e0aef2a05d47d870af640fe7e4ad721 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Fri, 24 Mar 2023 12:41:20 +0100 Subject: [PATCH 34/58] comments addressed --- manta-accounting/src/wallet/mod.rs | 74 +++++++++-------------- manta-accounting/src/wallet/signer/mod.rs | 20 +++++- manta-crypto/src/accumulator.rs | 6 +- manta-crypto/src/merkle_tree/forest.rs | 4 +- 4 files changed, 50 insertions(+), 54 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 9c8909f04..a1bd650c2 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -40,7 +40,7 @@ use crate::{ BalanceUpdate, Checkpoint, IdentityRequest, IdentityResponse, InitialSyncData, InitialSyncRequest, SignError, SignRequest, SignResponse, SignWithTransactionDataResponse, SyncData, SyncError, SyncRequest, SyncResponse, - TransactionDataRequest, TransactionDataResponse, + SyncResult, TransactionDataRequest, TransactionDataResponse, }, }, }; @@ -300,6 +300,11 @@ where let mut is_continue = true; let mut checkpoint = Default::default(); let mut request = InitialSyncRequest::::default(); + let parameters = self + .signer + .transfer_parameters() + .await + .map_err(Error::SignerConnectionError)?; while is_continue { let ReadResponse { should_continue, @@ -310,14 +315,7 @@ where .await .map_err(Error::LedgerConnectionError)?; is_continue = should_continue; - request.extend_with_data( - &self - .signer - .transfer_parameters() - .await - .map_err(Error::SignerConnectionError)?, - data, - ); + request.extend_with_data(¶meters, data); checkpoint .update_from_utxo_count(request.utxo_data.iter().map(|utxos| utxos.len()).collect()) } @@ -365,18 +363,13 @@ where Ok(ControlFlow::should_continue(should_continue)) } - /// Performs a synchronization with the signer against the given `request`. + /// Updates `self` from `response`. #[inline] - async fn signer_sync( + fn process_sync_response( &mut self, - request: SyncRequest, + response: SyncResult, ) -> Result<(), Error> { - match self - .signer - .sync(request) - .await - .map_err(Error::SignerConnectionError)? - { + match response { Ok(SyncResponse { checkpoint, balance_update, @@ -410,43 +403,32 @@ where } } + /// Performs a synchronization with the signer against the given `request`. + #[inline] + async fn signer_sync( + &mut self, + request: SyncRequest, + ) -> Result<(), Error> { + let response = self + .signer + .sync(request) + .await + .map_err(Error::SignerConnectionError)?; + self.process_sync_response(response) + } + /// Performs an initial synchronization with the signer against the given `request`. #[inline] async fn signer_initial_sync( &mut self, request: InitialSyncRequest, ) -> Result<(), Error> { - match self + let response = self .signer .initial_sync(request) .await - .map_err(Error::SignerConnectionError)? - { - Ok(SyncResponse { - checkpoint, - balance_update, - }) => { - match balance_update { - BalanceUpdate::Full { assets } => { - self.assets.clear(); - self.assets.deposit_all(assets); - } - _ => { - unreachable!("No transactions could have happened on a new account."); - } - } - self.checkpoint = checkpoint; - Ok(()) - } - Err(SyncError::InconsistentSynchronization { checkpoint: _ }) => { - unreachable!("Initial synchronization always starts at the default checkpoint.") - } - _ => { - unreachable!( - "Proof authorization key is not required for the initial synchronization." - ); - } - } + .map_err(Error::SignerConnectionError)?; + self.process_sync_response(response) } /// Checks if `transaction` can be executed on the balance state of `self`, returning the diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index c54f00e95..13fd6259f 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -166,6 +166,11 @@ where pub utxo_data: Vec>, /// Membership Proof Data + /// + /// # Note + /// + /// Each [`UtxoAccumulatorWitness`] here is the membership proof of the last item + /// of each subaccumulator in the [`UtxoAccumulator`](Configuration::UtxoAccumulator). pub membership_proof_data: Vec>, /// Nullifier Count @@ -217,6 +222,11 @@ where pub utxo_data: Vec>>, /// Membership Proof Data + /// + /// # Note + /// + /// Each [`UtxoAccumulatorWitness`] here is the membership proof of the last item + /// of each subaccumulator in the [`UtxoAccumulator`](Configuration::UtxoAccumulator). pub membership_proof_data: Vec>, /// Nullifier Count @@ -279,8 +289,14 @@ where fn default() -> Self { let mut utxo_data = Vec::>::default(); let mut membership_proof_data = Vec::new(); - utxo_data.resize_with(C::UtxoAccumulator::NUMBER_OF_PROOFS, Default::default); - membership_proof_data.resize_with(C::UtxoAccumulator::NUMBER_OF_PROOFS, Default::default); + utxo_data.resize_with( + C::UtxoAccumulator::NUMBER_OF_SUBACCUMULATORS, + Default::default, + ); + membership_proof_data.resize_with( + C::UtxoAccumulator::NUMBER_OF_SUBACCUMULATORS, + Default::default, + ); Self { utxo_data, membership_proof_data, diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 1f0bd9bec..9b081c42f 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -205,8 +205,8 @@ pub trait OptimizedAccumulator: Accumulator { /// From Items and Witnesses pub trait FromItemsAndWitnesses: Accumulator { - /// Number of Proofs - const NUMBER_OF_PROOFS: usize; + /// Number of Subaccumulators + const NUMBER_OF_SUBACCUMULATORS: usize; /// Builds a new [`Self`] from `items` and `proofs`. fn from_items_and_witnesses( @@ -215,7 +215,7 @@ pub trait FromItemsAndWitnesses: Accumulator { witnesses: Vec, ) -> Self; - /// Sorts `items`. + /// Groups `items` by the subaccumulator they belong to. // TODO: move this to the model. fn sort_items(items: Vec) -> Vec>; } diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index e8675b4d4..e0596067f 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -567,7 +567,7 @@ macro_rules! impl_from_items_and_witnesses { LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { - const NUMBER_OF_PROOFS: usize = N; + const NUMBER_OF_SUBACCUMULATORS: usize = N; #[inline] fn from_items_and_witnesses( @@ -586,12 +586,10 @@ macro_rules! impl_from_items_and_witnesses { fn sort_items(items: Vec) -> Vec> { let mut result = Vec::>::default(); result.resize_with(N, Default::default); - for item in items { let tree_index = C::tree_index(&item).into(); result[tree_index].push(item); } - result } } From e76bb56576ce7418ad884960016021a88efaf4e9 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Fri, 24 Mar 2023 19:52:47 +0100 Subject: [PATCH 35/58] inner tree logic --- manta-crypto/src/merkle_tree/inner_tree.rs | 258 ++++++++++++++++++- manta-crypto/src/merkle_tree/node.rs | 79 +++++- manta-crypto/src/merkle_tree/test/mod.rs | 3 + manta-crypto/src/merkle_tree/test/partial.rs | 2 +- 4 files changed, 337 insertions(+), 5 deletions(-) diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index 7e769dcaf..79d622c07 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -22,10 +22,16 @@ use crate::merkle_tree::{ path::{CurrentInnerPath, InnerPath}, - path_length, Configuration, InnerDigest, Node, Parameters, Parity, + path_length, Configuration, DualParity, InnerDigest, Node, NodeRange, Parameters, Parity, +}; +use alloc::{collections::btree_map, vec::Vec}; +use core::{ + fmt::Debug, + hash::Hash, + iter::FusedIterator, + marker::PhantomData, + ops::{Index, Range}, }; -use alloc::collections::btree_map; -use core::{fmt::Debug, hash::Hash, iter::FusedIterator, marker::PhantomData, ops::Index}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; @@ -199,6 +205,177 @@ impl ExactSizeIterator for InnerNodeIter {} impl FusedIterator for InnerNodeIter {} +/// +#[cfg_attr( + feature = "serde", + derive(Deserialize, Serialize), + serde(crate = "manta_util::serde", deny_unknown_fields) +)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub struct InnerNodeRange { + /// + depth: usize, + + /// + node_range: NodeRange, +} + +impl InnerNodeRange { + /// + #[inline] + const fn new_unchecked(depth: usize, node_range: NodeRange) -> Self { + Self { depth, node_range } + } + + /// + #[inline] + pub const fn new(depth: usize, node_range: NodeRange) -> Option { + let last_node = node_range.last_node(); + if last_node.0 >= (1 << (depth + 1)) { + None + } else { + Some(Self::new_unchecked(depth, node_range)) + } + } + + /// + #[inline] + pub fn from_leaves(leaf_index: Node, extra_leaves: usize) -> Option + where + C: Configuration + ?Sized, + { + Self::new( + path_length::(), + NodeRange { + node: leaf_index, + extra_nodes: extra_leaves, + }, + ) + .map(|inner_node_range| inner_node_range.parents()) + .flatten() + } + + /// + #[inline] + pub const fn dual_parity(&self) -> DualParity { + self.node_range.dual_parity() + } + + /// + #[inline] + pub const fn parents(&self) -> Option { + match self.depth.checked_sub(1) { + Some(depth) => Some(Self::new_unchecked(depth, self.node_range.parents())), + _ => None, + } + } + + /// Converts `self` into its parent, if the parent exists, returning the parent [`InnerNode`]. + #[inline] + pub fn into_parents(&mut self) -> Option { + match self.parents() { + Some(parents) => { + *self = parents; + Some(*self) + } + _ => None, + } + } + + /// + #[inline] + pub const fn depth(&self) -> usize { + self.depth + } + + /// + #[inline] + pub const fn node_range(&self) -> NodeRange { + self.node_range + } + + /// + #[inline] + pub const fn starting_inner_node(&self) -> InnerNode { + InnerNode { + depth: self.depth, + index: self.node_range.node, + } + } + + /// + #[inline] + pub const fn last_inner_node(&self) -> InnerNode { + InnerNode { + depth: self.depth, + index: self.node_range.last_node(), + } + } + + /// + #[inline] + pub fn map_indices(&self) -> Range { + self.starting_inner_node().map_index()..self.last_inner_node().map_index() + 1 + } +} + +/// InnerNodeRangeIter +#[cfg_attr( + feature = "serde", + derive(Deserialize, Serialize), + serde(crate = "manta_util::serde", deny_unknown_fields) +)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +pub struct InnerNodeRangeIter { + /// + node_range: Option, +} + +impl InnerNodeRangeIter { + /// Builds a new [`InnerNodeRangeIter`] from `node_range`. + #[inline] + const fn new(node_range: Option) -> Self { + Self { node_range } + } + + /// Builds a new [`InnerNodeIter`] iterator over the parents of `leaf_index`. + #[inline] + pub fn from_leaves(leaf_index: Node, extra_leaves: usize) -> Self + where + C: Configuration + ?Sized, + { + Self::new(InnerNodeRange::from_leaves::(leaf_index, extra_leaves)) + } + + /// Returns `true` if the iterator has completed. + #[inline] + pub const fn is_done(&self) -> bool { + self.node_range.is_none() + } +} + +// TODO: Add all methods which can be optimized. +impl Iterator for InnerNodeRangeIter { + type Item = InnerNodeRange; + + #[inline] + fn next(&mut self) -> Option { + let node_range = self.node_range.take()?; + self.node_range = node_range.parents(); + Some(node_range) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.node_range.map_or(0, move |n| n.depth + 1); + (len, Some(len)) + } +} + +impl ExactSizeIterator for InnerNodeRangeIter {} + +impl FusedIterator for InnerNodeRangeIter {} + /// [`InnerTree`] Map Backend pub trait InnerMap where @@ -527,6 +704,41 @@ where parameters.join(lhs, rhs) } + /// + #[inline] + fn insert_range_and_join( + &mut self, + parameters: &Parameters, + node: InnerNodeRange, + mut inner_digests: Vec>, + ) -> Vec> { + let indices = node.map_indices(); + let mut result = Vec::new(); + for index in indices.clone() { + self.map + .set(index, inner_digests.pop().expect("Missing inner digest.")); + } + match node.dual_parity().0 { + (Parity::Left, _) => { + for index in indices.step_by(2) { + result.push(parameters.join( + self.map_get_or_sentinel(index), + self.map_get_or_sentinel(index + 1), + )); + } + } + (Parity::Right, _) => { + for index in indices.step_by(2) { + result.push(parameters.join( + self.map_get_or_sentinel(index - 1), + self.map_get_or_sentinel(index), + )); + } + } + } + result + } + /// Computes the new root of the tree after inserting `base` which corresponds to the leaf at /// `leaf_index`. #[inline] @@ -541,6 +753,35 @@ where }) } + /// + #[inline] + fn compute_root_from_leaves( + &mut self, + parameters: &Parameters, + leaf_indices: NodeRange, + base: Vec>, + ) -> InnerDigest { + InnerNodeRangeIter::from_leaves::(leaf_indices.node, leaf_indices.extra_nodes) + .fold(base, move |inner_digests, node| { + self.insert_range_and_join(parameters, node, inner_digests) + }) + .into_iter() + .next() + .expect("The final vector consists of the root of the Merkle tree.") + } + + /// + #[inline] + pub fn batch_insert( + &mut self, + parameters: &Parameters, + leaf_indices: NodeRange, + base: Vec>, + ) { + let root = self.compute_root_from_leaves(parameters, leaf_indices, base); + self.set_root(root) + } + /// Inserts the `base` inner digest corresponding to the leaf at `leaf_index` into the tree. #[inline] pub fn insert(&mut self, parameters: &Parameters, leaf_index: Node, base: InnerDigest) { @@ -758,6 +999,17 @@ where self.inner_tree.insert(parameters, leaf_index, base); } + /// + #[inline] + pub fn batch_insert( + &mut self, + parameters: &Parameters, + leaf_indices: NodeRange, + base: Vec>, + ) { + self.inner_tree.batch_insert(parameters, leaf_indices, base); + } + /// Computes the inner path of the leaf given by `leaf_index` without checking if /// `leaf_index` is later than the starting index of this tree. #[inline] diff --git a/manta-crypto/src/merkle_tree/node.rs b/manta-crypto/src/merkle_tree/node.rs index a511744e1..2fb4bd810 100644 --- a/manta-crypto/src/merkle_tree/node.rs +++ b/manta-crypto/src/merkle_tree/node.rs @@ -159,7 +159,7 @@ impl Default for Parity { derive(Deserialize, Serialize), serde(crate = "manta_util::serde", deny_unknown_fields) )] -#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct Node( /// Level-wise Index to a node in a Binary Tree pub Idx, @@ -262,6 +262,12 @@ impl Node { Self(self.0 >> 1) } + /// Returns the `k`-th ancestor [`Node`] of `self`. + #[inline] + pub const fn ancestor(&self, k: usize) -> Self { + Self(self.0 >> k) + } + /// Converts `self` into its parent, returning the parent [`Node`]. #[inline] #[must_use] @@ -433,3 +439,74 @@ impl Iterator for NodeParents { } impl FusedIterator for NodeParents {} + +/// Dual Parity +#[cfg_attr( + feature = "serde", + derive(Deserialize, Serialize), + serde(crate = "manta_util::serde", deny_unknown_fields) +)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct DualParity(pub (Parity, Parity)); + +impl DualParity { + /// + #[inline] + pub const fn starting_parity(&self) -> Parity { + self.0 .0 + } + + /// + #[inline] + pub const fn final_parity(&self) -> Parity { + self.0 .1 + } +} + +/// Node Range +#[cfg_attr( + feature = "serde", + derive(Deserialize, Serialize), + serde(crate = "manta_util::serde", deny_unknown_fields) +)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct NodeRange { + /// Starting Node + pub node: Node, + + /// Extra Nodes + pub extra_nodes: usize, +} + +impl NodeRange { + /// + #[inline] + pub const fn dual_parity(&self) -> DualParity { + let starting_node_parity = self.node.parity(); + let final_node_parity = match starting_node_parity { + Parity::Left => Parity::from_index(self.extra_nodes), + Parity::Right => Parity::from_index(self.extra_nodes + 1), + }; + DualParity((starting_node_parity, final_node_parity)) + } + + /// + #[inline] + pub const fn parents(&self) -> Self { + let extra_nodes = match self.dual_parity().0 { + (Parity::Left, Parity::Right) => (self.extra_nodes - 1) >> 1, + (Parity::Right, Parity::Left) => (self.extra_nodes + 1) >> 1, + _ => self.extra_nodes >> 1, + }; + Self { + node: self.node.parent(), + extra_nodes, + } + } + + /// + #[inline] + pub const fn last_node(&self) -> Node { + Node(self.node.0 + self.extra_nodes) + } +} diff --git a/manta-crypto/src/merkle_tree/test/mod.rs b/manta-crypto/src/merkle_tree/test/mod.rs index d13720a15..c7148f634 100644 --- a/manta-crypto/src/merkle_tree/test/mod.rs +++ b/manta-crypto/src/merkle_tree/test/mod.rs @@ -27,6 +27,9 @@ use crate::{ use alloc::string::String; use core::{fmt::Debug, hash::Hash, marker::PhantomData}; +#[cfg(test)] +pub mod node; + #[cfg(test)] pub mod partial; diff --git a/manta-crypto/src/merkle_tree/test/partial.rs b/manta-crypto/src/merkle_tree/test/partial.rs index 332d8cc69..3d85a7a25 100644 --- a/manta-crypto/src/merkle_tree/test/partial.rs +++ b/manta-crypto/src/merkle_tree/test/partial.rs @@ -51,7 +51,7 @@ fn test_from_leaves_and_path() { } let forked_tree = ForkedTree::>::new(tree.tree.clone(), ¶meters); let path = tree.current_path(); - let partial_tree = PartialMerkleTree::> { + let partial_tree = PartialMerkleTree:: { parameters, tree: Partial::from_leaves_and_path_unchecked( ¶meters, From b5d5bab1ee3333c82be78d75b4fb0cf908828c38 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 27 Mar 2023 20:48:35 +0200 Subject: [PATCH 36/58] all trees take batch insertions --- manta-crypto/src/accumulator.rs | 30 +++++ manta-crypto/src/merkle_tree/fork.rs | 69 +++++++++++ manta-crypto/src/merkle_tree/full.rs | 61 ++++++++++ manta-crypto/src/merkle_tree/inner_tree.rs | 3 +- manta-crypto/src/merkle_tree/node.rs | 41 +++++++ manta-crypto/src/merkle_tree/partial.rs | 76 +++++++++++- manta-crypto/src/merkle_tree/tree.rs | 134 +++++++++++++++++++++ 7 files changed, 411 insertions(+), 3 deletions(-) diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 9b081c42f..88af72a79 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -145,6 +145,21 @@ pub trait Accumulator: Types { fn contains(&self, item: &Self::Item) -> bool { self.prove(item).is_some() } + + /// + #[inline] + fn batch_insert<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + for item in items.into_iter() { + if !self.insert(item) { + return false; + } + } + true + } } /// Constant Capacity Accumulator @@ -185,6 +200,21 @@ pub trait OptimizedAccumulator: Accumulator { self.insert(item) } + /// + #[inline] + fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + for item in items.into_iter() { + if !self.insert_nonprovable(item) { + return false; + } + } + true + } + /// Removes the witnesses to the membership of `item` in `self`. The resulting state of the /// accumulator after removing a proof should be the same as if the item had been inserted into /// the accumulator with [`insert_nonprovable`](Self::insert_nonprovable). This method returns diff --git a/manta-crypto/src/merkle_tree/fork.rs b/manta-crypto/src/merkle_tree/fork.rs index b4a44b957..3ced19328 100644 --- a/manta-crypto/src/merkle_tree/fork.rs +++ b/manta-crypto/src/merkle_tree/fork.rs @@ -521,6 +521,20 @@ where self.data.push(parameters, leaf) } + /// + #[inline] + fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + LeafDigest: Default, + { + self.data.batch_maybe_push_digest(parameters, leaf_digests) + } + /// Appends a new `leaf_digest` onto this branch. #[inline] fn maybe_push_digest(&mut self, parameters: &Parameters, leaf_digest: F) -> Option @@ -758,6 +772,22 @@ where self.check_attachment()?; self.branch.maybe_push_digest(parameters, leaf_digest) } + + /// + #[inline] + pub fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + LeafDigest: Default, + { + self.check_attachment()?; + self.branch + .batch_maybe_push_digest(parameters, leaf_digests) + } } /// Forked Tree @@ -908,6 +938,21 @@ where self.branch.maybe_push_digest(parameters, leaf_digest) } + /// + #[inline] + fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + LeafDigest: Default, + { + self.branch + .batch_maybe_push_digest(parameters, leaf_digests) + } + /// Resets the fork of the base tree back to the trunk. #[inline] pub fn reset_fork(&mut self, parameters: &Parameters) @@ -992,6 +1037,18 @@ where { self.maybe_push_digest(parameters, leaf_digest) } + + #[inline] + fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_digest(parameters, leaf_digests) + } } impl WithProofs for ForkedTree @@ -1028,4 +1085,16 @@ where fn path(&self, parameters: &Parameters, index: usize) -> Result, PathError> { self.path(parameters, index) } + + #[inline] + fn batch_maybe_push_provable_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_digest(parameters, leaf_digests) + } } diff --git a/manta-crypto/src/merkle_tree/full.rs b/manta-crypto/src/merkle_tree/full.rs index 97ee30d85..866e4c9d9 100644 --- a/manta-crypto/src/merkle_tree/full.rs +++ b/manta-crypto/src/merkle_tree/full.rs @@ -30,6 +30,8 @@ use core::{fmt::Debug, hash::Hash}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; +use super::NodeRange; + /// Full Merkle Tree Type pub type FullMerkleTree> = MerkleTree>; @@ -153,6 +155,24 @@ where ); self.leaf_digests.push(leaf_digest); } + + /// + #[inline] + fn batch_push_leaf_digests( + &mut self, + parameters: &Parameters, + leaf_indices: NodeRange, + leaf_digests: Vec>, + ) where + LeafDigest: Default, + { + let base_inner_digests = leaf_indices.join_leaves(parameters, &leaf_digests, |node| { + self.get_leaf_sibling(node) + }); + self.inner_digests + .batch_insert(parameters, leaf_indices, base_inner_digests); + self.leaf_digests.extend(leaf_digests); + } } impl Tree for Full @@ -209,6 +229,35 @@ where self.push_leaf_digest(parameters, Node(len), leaf_digest()?); Some(true) } + + #[inline] + fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + let len = self.len(); + let leaf_digests = leaf_digests(); + let number_of_leaf_digests = leaf_digests.len(); + if leaf_digests.is_empty() { + return None; + } + if len + number_of_leaf_digests > capacity::() { + return Some(false); + } + self.batch_push_leaf_digests( + parameters, + NodeRange { + node: Node(len), + extra_nodes: number_of_leaf_digests - 1, + }, + leaf_digests, + ); + Some(true) + } } impl WithProofs for Full @@ -261,4 +310,16 @@ where let _ = index; false } + + #[inline] + fn batch_maybe_push_provable_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_digest(parameters, leaf_digests) + } } diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index 79d622c07..893e603bd 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -251,8 +251,7 @@ impl InnerNodeRange { extra_nodes: extra_leaves, }, ) - .map(|inner_node_range| inner_node_range.parents()) - .flatten() + .and_then(|inner_node_range| inner_node_range.parents()) } /// diff --git a/manta-crypto/src/merkle_tree/node.rs b/manta-crypto/src/merkle_tree/node.rs index 2fb4bd810..995e42304 100644 --- a/manta-crypto/src/merkle_tree/node.rs +++ b/manta-crypto/src/merkle_tree/node.rs @@ -17,6 +17,7 @@ //! Merkle Tree Node Abstractions use crate::merkle_tree::{HashConfiguration, InnerDigest, InnerHash, LeafDigest, Parameters}; +use alloc::vec::Vec; use core::{ iter::FusedIterator, ops::{Add, Sub}, @@ -509,4 +510,44 @@ impl NodeRange { pub const fn last_node(&self) -> Node { Node(self.node.0 + self.extra_nodes) } + + /// + #[inline] + pub fn join_leaves<'a, C, F>( + &self, + parameters: &Parameters, + leaves: &Vec>, + mut get_leaves: F, + ) -> Vec> + where + C: HashConfiguration + ?Sized, + LeafDigest: 'a + Default, + F: FnMut(Node) -> Option<&'a LeafDigest>, + { + let dual_parity = self.dual_parity(); + let mut result = Vec::new(); + let length = leaves.len(); + let range = match dual_parity.starting_parity() { + Parity::Left => (0..length - 1).step_by(2), + _ => { + result.push(Node(0).join_leaves( + parameters, + get_leaves(self.node).unwrap_or(&Default::default()), + &leaves[0], + )); + (1..length - 1).step_by(2) + } + }; + for i in range { + result.push(Node(i).join_leaves(parameters, &leaves[i], &leaves[i + 1])) + } + if dual_parity.final_parity().is_left() { + result.push(self.last_node().join_leaves( + parameters, + &leaves[length - 1], + get_leaves(self.last_node()).unwrap_or(&Default::default()), + )) + } + result + } } diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index 806bb7ed2..6b43bccfe 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -21,7 +21,7 @@ use crate::merkle_tree::{ capacity, inner_tree::{BTreeMap, InnerMap, PartialInnerTree}, - node::Parity, + node::{NodeRange, Parity}, Configuration, CurrentPath, InnerDigest, Leaf, LeafDigest, MerkleTree, Node, Parameters, Path, PathError, Root, Tree, WithProofs, }; @@ -269,6 +269,56 @@ where self.leaf_digests.push(leaf_digest); } + /// + #[inline] + fn batch_push_leaf_digests( + &mut self, + parameters: &Parameters, + leaf_indices: NodeRange, + leaf_digests: Vec>, + ) where + LeafDigest: Default, + { + let base_inner_digests = leaf_indices.join_leaves(parameters, &leaf_digests, |node| { + self.get_leaf_sibling(node) + }); + self.inner_digests + .batch_insert(parameters, leaf_indices, base_inner_digests); + self.leaf_digests.extend(leaf_digests); + } + + /// + #[inline] + pub fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + LeafDigest: Default, + { + // TODO: Push without keeping unnecessary proof. + let len = self.len(); + let leaf_digests = leaf_digests(); + let number_of_leaf_digests = leaf_digests.len(); + if leaf_digests.is_empty() { + return None; + } + if len + number_of_leaf_digests > capacity::() { + return Some(false); + } + self.batch_push_leaf_digests( + parameters, + NodeRange { + node: Node(len), + extra_nodes: number_of_leaf_digests - 1, + }, + leaf_digests, + ); + Some(true) + } + /// Appends a `leaf` to the tree using `parameters`. #[inline] pub fn push(&mut self, parameters: &Parameters, leaf: &Leaf) -> bool @@ -345,6 +395,18 @@ where { self.maybe_push_digest(parameters, leaf_digest) } + + #[inline] + fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_digest(parameters, leaf_digests) + } } impl WithProofs for Partial @@ -395,4 +457,16 @@ where let _ = index; false } + + #[inline] + fn batch_maybe_push_provable_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_digest(parameters, leaf_digests) + } } diff --git a/manta-crypto/src/merkle_tree/tree.rs b/manta-crypto/src/merkle_tree/tree.rs index 3a25508da..0d2f04f8e 100644 --- a/manta-crypto/src/merkle_tree/tree.rs +++ b/manta-crypto/src/merkle_tree/tree.rs @@ -41,6 +41,7 @@ use crate::{ path::{constraint::PathVar, CurrentPath, Path}, }, }; +use alloc::vec::Vec; use core::{fmt::Debug, hash::Hash, marker::PhantomData}; use manta_util::{ codec::{Decode, DecodeError, Encode, Read, Write}, @@ -301,6 +302,53 @@ where self.push_digest(parameters, || parameters.digest(leaf)) } + /// + #[inline] + fn batch_maybe_push_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + let leaf_digests = leaf_digests(); + if leaf_digests.is_empty() { + return None; + } + for leaf_digest in leaf_digests { + if !self.maybe_push_digest(parameters, || Some(leaf_digest))? { + return Some(false); + } + } + Some(true) + } + + /// + #[inline] + fn batch_push_digest(&mut self, parameters: &Parameters, leaf_digests: F) -> bool + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_digest(parameters, leaf_digests) + .unwrap() + } + + /// + #[inline] + fn batch_push<'a, I>(&mut self, parameters: &Parameters, leaves: I) -> bool + where + Leaf: 'a, + I: IntoIterator>, + { + self.batch_push_digest(parameters, || { + leaves + .into_iter() + .map(|leaf| parameters.digest(leaf)) + .collect() + }) + } + /// Appends an iterator of leaf digests at the end of the tree, returning the iterator back /// if it could not be inserted because the tree has exhausted its capacity. /// @@ -451,6 +499,53 @@ where self.push_provable_digest(parameters, move || parameters.digest(leaf)) } + /// + #[inline] + fn batch_maybe_push_provable_digest( + &mut self, + parameters: &Parameters, + leaf_digests: F, + ) -> Option + where + F: FnOnce() -> Vec>, + { + let leaf_digests = leaf_digests(); + if leaf_digests.is_empty() { + return None; + } + for leaf_digest in leaf_digests { + if !self.maybe_push_provable_digest(parameters, || Some(leaf_digest))? { + return Some(false); + } + } + Some(true) + } + + /// + #[inline] + fn batch_push_provable_digest(&mut self, parameters: &Parameters, leaf_digests: F) -> bool + where + F: FnOnce() -> Vec>, + { + self.batch_maybe_push_provable_digest(parameters, leaf_digests) + .unwrap() + } + + /// + #[inline] + fn batch_push_provable<'a, I>(&mut self, parameters: &Parameters, leaves: I) -> bool + where + Leaf: 'a, + I: IntoIterator>, + { + self.batch_push_provable_digest(parameters, || { + leaves + .into_iter() + .map(|leaf| parameters.digest(leaf)) + .collect() + }) + } + /// Returns the path for the leaf stored at the given `index` if it exists. fn path(&self, parameters: &Parameters, index: usize) -> Result, PathError>; @@ -913,6 +1008,16 @@ where self.tree.push(&self.parameters, leaf) } + /// + #[inline] + pub fn batch_push<'a, I>(&mut self, leaves: I) -> bool + where + Leaf: 'a, + I: IntoIterator>, + { + self.tree.batch_push(&self.parameters, leaves) + } + /// Appends an iterator of leaves at the end of the tree, returning `false` if the `leaves` /// could not be inserted because the tree has exhausted its capacity. /// @@ -995,6 +1100,17 @@ where self.tree.push_provable(&self.parameters, leaf) } + /// + #[inline] + pub fn batch_push_provable<'a, I>(&mut self, leaves: I) -> bool + where + T: WithProofs, + Leaf: 'a, + I: IntoIterator>, + { + self.tree.batch_push_provable(&self.parameters, leaves) + } + /// Returns the path for the leaf stored at the given `index` if it exists. /// /// See [`WithProofs::path`] for more. @@ -1099,6 +1215,15 @@ where fn contains(&self, item: &Self::Item) -> bool { self.contains(&self.parameters.digest(item)) } + + #[inline] + fn batch_insert<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + self.batch_push_provable(items) + } } impl ConstantCapacityAccumulator for MerkleTree @@ -1151,6 +1276,15 @@ where .map(move |i| self.tree.remove_path(i)) .unwrap_or(false) } + + #[inline] + fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + self.batch_push(items) + } } impl Rollback for MerkleTree> From 840b0cc5975dcd1397a9a10622b29d3d12983afc Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 14:47:45 +0200 Subject: [PATCH 37/58] impl for forest --- manta-crypto/src/accumulator.rs | 63 ++++++++++++++------------ manta-crypto/src/merkle_tree/forest.rs | 46 ++++++++++++++++++- manta-crypto/src/merkle_tree/tree.rs | 30 +++++++----- 3 files changed, 96 insertions(+), 43 deletions(-) diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 88af72a79..883901eb6 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -145,21 +145,6 @@ pub trait Accumulator: Types { fn contains(&self, item: &Self::Item) -> bool { self.prove(item).is_some() } - - /// - #[inline] - fn batch_insert<'a, I>(&mut self, items: I) -> bool - where - Self::Item: 'a, - I: IntoIterator, - { - for item in items.into_iter() { - if !self.insert(item) { - return false; - } - } - true - } } /// Constant Capacity Accumulator @@ -200,21 +185,6 @@ pub trait OptimizedAccumulator: Accumulator { self.insert(item) } - /// - #[inline] - fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool - where - Self::Item: 'a, - I: IntoIterator, - { - for item in items.into_iter() { - if !self.insert_nonprovable(item) { - return false; - } - } - true - } - /// Removes the witnesses to the membership of `item` in `self`. The resulting state of the /// accumulator after removing a proof should be the same as if the item had been inserted into /// the accumulator with [`insert_nonprovable`](Self::insert_nonprovable). This method returns @@ -233,6 +203,39 @@ pub trait OptimizedAccumulator: Accumulator { } } +/// Batch Insertion +pub trait BatchInsertion: OptimizedAccumulator { + /// + #[inline] + fn batch_insert<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + for item in items.into_iter() { + if !self.insert(item) { + return false; + } + } + true + } + + /// + #[inline] + fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + for item in items.into_iter() { + if !self.insert_nonprovable(item) { + return false; + } + } + true + } +} + /// From Items and Witnesses pub trait FromItemsAndWitnesses: Accumulator { /// Number of Subaccumulators diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index e0596067f..788c80ef9 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -24,7 +24,7 @@ use crate::{ accumulator::{ - self, Accumulator, ConstantCapacityAccumulator, ExactSizeAccumulator, + self, Accumulator, BatchInsertion, ConstantCapacityAccumulator, ExactSizeAccumulator, FromItemsAndWitnesses, MembershipProof, OptimizedAccumulator, }, merkle_tree::{ @@ -37,7 +37,7 @@ use crate::{ }, }; use alloc::{boxed::Box, vec::Vec}; -use core::{fmt::Debug, hash::Hash, marker::PhantomData}; +use core::{fmt::Debug, hash::Hash, iter::Iterator, marker::PhantomData}; use manta_util::{persistence::Rollback, BoxArray}; #[cfg(feature = "serde")] @@ -593,6 +593,48 @@ macro_rules! impl_from_items_and_witnesses { result } } + + impl BatchInsertion for $forest + where + C: Configuration + ?Sized, + C::Index: FixedIndex, + Parameters: Clone, + Leaf: Clone, + LeafDigest: Clone + Default + PartialEq, + InnerDigest: Clone + Default + PartialEq, + { + /// + #[inline] + fn batch_insert<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + let grouped_items = Self::sort_items(items.into_iter().cloned().collect()); + let mut result = true; + for (index, group) in grouped_items.into_iter().enumerate() { + let tree = self.forest.get_mut(C::Index::from_index(index)); + result &= tree.batch_push_provable(&self.parameters, &group); + } + result + } + + /// + #[inline] + fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + let grouped_items = Self::sort_items(items.into_iter().cloned().collect()); + let mut result = true; + for (index, group) in grouped_items.into_iter().enumerate() { + let tree = self.forest.get_mut(C::Index::from_index(index)); + result &= tree.batch_push(&self.parameters, &group); + } + result + } + } }; } diff --git a/manta-crypto/src/merkle_tree/tree.rs b/manta-crypto/src/merkle_tree/tree.rs index 0d2f04f8e..18e24dee4 100644 --- a/manta-crypto/src/merkle_tree/tree.rs +++ b/manta-crypto/src/merkle_tree/tree.rs @@ -26,8 +26,8 @@ use crate::{ accumulator::{ - self, Accumulator, ConstantCapacityAccumulator, ExactSizeAccumulator, MembershipProof, - OptimizedAccumulator, + self, Accumulator, BatchInsertion, ConstantCapacityAccumulator, ExactSizeAccumulator, + MembershipProof, OptimizedAccumulator, }, eclair::{ self, @@ -1215,15 +1215,6 @@ where fn contains(&self, item: &Self::Item) -> bool { self.contains(&self.parameters.digest(item)) } - - #[inline] - fn batch_insert<'a, I>(&mut self, items: I) -> bool - where - Self::Item: 'a, - I: IntoIterator, - { - self.batch_push_provable(items) - } } impl ConstantCapacityAccumulator for MerkleTree @@ -1276,6 +1267,23 @@ where .map(move |i| self.tree.remove_path(i)) .unwrap_or(false) } +} + +impl BatchInsertion for MerkleTree +where + C: Configuration + ?Sized, + T: Tree + WithProofs, + InnerDigest: Clone + PartialEq, + Parameters: Clone, +{ + #[inline] + fn batch_insert<'a, I>(&mut self, items: I) -> bool + where + Self::Item: 'a, + I: IntoIterator, + { + self.batch_push_provable(items) + } #[inline] fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool From ba4a6d8d28ab701a5abe438cd65555883589c86c Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 17:42:25 +0200 Subject: [PATCH 38/58] bug fixed --- manta-crypto/src/merkle_tree/forest.rs | 6 ++- manta-crypto/src/merkle_tree/fork.rs | 11 ++++ manta-crypto/src/merkle_tree/inner_tree.rs | 6 +-- manta-crypto/src/merkle_tree/partial.rs | 11 +++- .../src/merkle_tree/test/batch_insertion.rs | 51 +++++++++++++++++++ manta-crypto/src/merkle_tree/test/mod.rs | 2 +- manta-crypto/src/merkle_tree/test/partial.rs | 11 ++-- manta-crypto/src/merkle_tree/tree.rs | 2 + 8 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 manta-crypto/src/merkle_tree/test/batch_insertion.rs diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 788c80ef9..407c4327f 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -534,6 +534,7 @@ macro_rules! impl_from_items_and_witnesses { C::Index: FixedIndex, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, { /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that /// the `paths` are consistent with the leaves and that they are @@ -566,6 +567,7 @@ macro_rules! impl_from_items_and_witnesses { Parameters: Clone, LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, { const NUMBER_OF_SUBACCUMULATORS: usize = N; @@ -601,7 +603,7 @@ macro_rules! impl_from_items_and_witnesses { Parameters: Clone, Leaf: Clone, LeafDigest: Clone + Default + PartialEq, - InnerDigest: Clone + Default + PartialEq, + InnerDigest: Clone + Default + PartialEq + Debug, { /// #[inline] @@ -771,6 +773,8 @@ where M: Default + InnerMap, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, + M: Debug { #[inline] fn rollback(&mut self) { diff --git a/manta-crypto/src/merkle_tree/fork.rs b/manta-crypto/src/merkle_tree/fork.rs index 3ced19328..881948f5b 100644 --- a/manta-crypto/src/merkle_tree/fork.rs +++ b/manta-crypto/src/merkle_tree/fork.rs @@ -531,6 +531,8 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, + InnerDigest: Debug, + M: Debug { self.data.batch_maybe_push_digest(parameters, leaf_digests) } @@ -783,6 +785,8 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, + InnerDigest: Debug, + M: Debug { self.check_attachment()?; self.branch @@ -948,6 +952,8 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, + InnerDigest: Debug, + M: Debug { self.branch .batch_maybe_push_digest(parameters, leaf_digests) @@ -980,6 +986,7 @@ where C: Configuration + ?Sized, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, { /// Builds a new [`ForkedTree`] from `leaf_digests` and `path` without checking that /// `path` is consistent with the leaves and that it is a [`CurrentPath`]. @@ -1003,6 +1010,8 @@ where M: Default + InnerMap, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, + M: Debug { #[inline] fn new(parameters: &Parameters) -> Self { @@ -1058,6 +1067,8 @@ where M: Default + InnerMap, LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default, + InnerDigest: Debug, + M: Debug { #[inline] fn leaf_digest(&self, index: usize) -> Option<&LeafDigest> { diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index 893e603bd..fbd160a8b 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -709,13 +709,13 @@ where &mut self, parameters: &Parameters, node: InnerNodeRange, - mut inner_digests: Vec>, + inner_digests: Vec>, ) -> Vec> { let indices = node.map_indices(); let mut result = Vec::new(); - for index in indices.clone() { + for (index, inner_digest) in indices.clone().zip(inner_digests) { self.map - .set(index, inner_digests.pop().expect("Missing inner digest.")); + .set(index, inner_digest); } match node.dual_parity().0 { (Parity::Left, _) => { diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index 6b43bccfe..3c922c74a 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -65,7 +65,7 @@ where leaf_digests: Vec>, /// Inner Digests - inner_digests: PartialInnerTree, + pub inner_digests: PartialInnerTree, } impl Partial @@ -278,10 +278,13 @@ where leaf_digests: Vec>, ) where LeafDigest: Default, + InnerDigest: Debug, + M: Debug { let base_inner_digests = leaf_indices.join_leaves(parameters, &leaf_digests, |node| { self.get_leaf_sibling(node) }); + println!("Inner digests: {:?}", base_inner_digests); self.inner_digests .batch_insert(parameters, leaf_indices, base_inner_digests); self.leaf_digests.extend(leaf_digests); @@ -297,6 +300,8 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, + InnerDigest: Debug, + M: Debug { // TODO: Push without keeping unnecessary proof. let len = self.len(); @@ -360,6 +365,8 @@ where M: InnerMap + Default, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, + M: Debug { #[inline] fn new(parameters: &Parameters) -> Self { @@ -415,6 +422,8 @@ where M: Default + InnerMap, LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, + M: Debug { #[inline] fn leaf_digest(&self, index: usize) -> Option<&LeafDigest> { diff --git a/manta-crypto/src/merkle_tree/test/batch_insertion.rs b/manta-crypto/src/merkle_tree/test/batch_insertion.rs new file mode 100644 index 000000000..7968f40bb --- /dev/null +++ b/manta-crypto/src/merkle_tree/test/batch_insertion.rs @@ -0,0 +1,51 @@ +// Copyright 2019-2022 Manta Network. +// This file is part of manta-rs. +// +// manta-rs is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// manta-rs is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with manta-rs. If not, see . + +//! Batch Insertion + +use crate::{ + accumulator::{Accumulator, BatchInsertion}, + merkle_tree::{ + inner_tree::{InnerNode, InnerNodeRangeIter}, node::Node, partial::PartialMerkleTree, test::Test, tree::Parameters, full::FullMerkleTree + }, + rand::{OsRng, Rand, Sample}, +}; + +/// Merkle Tree Height +const HEIGHT: usize = 4; + +/// Merkle Tree Configuration +type Config = Test; + +/// +#[test] +fn test_batch_insertion() { + let mut rng = OsRng; + let parameters = Parameters::::sample(Default::default(), &mut rng); + let mut tree = PartialMerkleTree::::new(parameters); + let mut cloned_tree = tree.clone(); + let insertions = vec!["a", "b", "c", "d", "e", "f", "g", "h"].into_iter().map(String::from).collect::>(); + for leaf in &insertions { + tree.insert(leaf); + } + println!("{insertions:?}"); + cloned_tree.batch_insert(&insertions); + let mut node = InnerNodeRangeIter::from_leaves::(0.into(), 7); + println!("{:?}", node.next()); + println!("{:?}", node.next()); + println!("{:?}", node.next()); + assert_eq!(tree.tree, cloned_tree.tree); +} \ No newline at end of file diff --git a/manta-crypto/src/merkle_tree/test/mod.rs b/manta-crypto/src/merkle_tree/test/mod.rs index c7148f634..35e065071 100644 --- a/manta-crypto/src/merkle_tree/test/mod.rs +++ b/manta-crypto/src/merkle_tree/test/mod.rs @@ -28,7 +28,7 @@ use alloc::string::String; use core::{fmt::Debug, hash::Hash, marker::PhantomData}; #[cfg(test)] -pub mod node; +pub mod batch_insertion; #[cfg(test)] pub mod partial; diff --git a/manta-crypto/src/merkle_tree/test/partial.rs b/manta-crypto/src/merkle_tree/test/partial.rs index 3d85a7a25..2120c975d 100644 --- a/manta-crypto/src/merkle_tree/test/partial.rs +++ b/manta-crypto/src/merkle_tree/test/partial.rs @@ -30,17 +30,20 @@ use crate::{ rand::{OsRng, Rand, Sample}, }; +/// Merkle Tree Height +const HEIGHT: usize = 7; + +/// Merkle Tree Configuration +type Config = Test; + /// Tests the [`Partial`] tree generated from a set of leaves and a [`Path`] behaves /// as expected. #[test] fn test_from_leaves_and_path() { let mut rng = OsRng; - const HEIGHT: usize = 7; - type Config = Test; let parameters = Parameters::::sample(Default::default(), &mut rng); let number_of_insertions = rng.gen_range(5..(1 << (HEIGHT - 1))); let inner_element_index = rng.gen_range(0..number_of_insertions - 3); - println!("{number_of_insertions}, {inner_element_index}"); let mut tree = FullMerkleTree::::new(parameters); let mut insertions = Vec::::with_capacity(number_of_insertions); for _ in 0..number_of_insertions { @@ -106,8 +109,6 @@ fn test_from_leaves_and_path() { #[test] fn test_from_leaves_and_path_forest() { let mut rng = OsRng; - const HEIGHT: usize = 7; - type Config = Test; let parameters = Parameters::::sample(Default::default(), &mut rng); let mut forest = TreeArrayMerkleForest::>, 2>::new(parameters); diff --git a/manta-crypto/src/merkle_tree/tree.rs b/manta-crypto/src/merkle_tree/tree.rs index 18e24dee4..c80fc20d2 100644 --- a/manta-crypto/src/merkle_tree/tree.rs +++ b/manta-crypto/src/merkle_tree/tree.rs @@ -1302,6 +1302,8 @@ where M: Default + InnerMap, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, + InnerDigest: Debug, + M: Debug { #[inline] fn rollback(&mut self) { From 1a358c740b08ceef0efcba8646569bcb154659df Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 18:32:47 +0200 Subject: [PATCH 39/58] test --- manta-crypto/src/merkle_tree/forest.rs | 2 +- manta-crypto/src/merkle_tree/fork.rs | 11 --- manta-crypto/src/merkle_tree/inner_tree.rs | 3 +- manta-crypto/src/merkle_tree/partial.rs | 12 +-- .../src/merkle_tree/test/batch_insertion.rs | 74 ++++++++++++++----- manta-crypto/src/merkle_tree/tree.rs | 2 - 6 files changed, 59 insertions(+), 45 deletions(-) diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 407c4327f..438c0b30d 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -774,7 +774,7 @@ where LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, InnerDigest: Debug, - M: Debug + M: Debug, { #[inline] fn rollback(&mut self) { diff --git a/manta-crypto/src/merkle_tree/fork.rs b/manta-crypto/src/merkle_tree/fork.rs index 881948f5b..3ced19328 100644 --- a/manta-crypto/src/merkle_tree/fork.rs +++ b/manta-crypto/src/merkle_tree/fork.rs @@ -531,8 +531,6 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, - InnerDigest: Debug, - M: Debug { self.data.batch_maybe_push_digest(parameters, leaf_digests) } @@ -785,8 +783,6 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, - InnerDigest: Debug, - M: Debug { self.check_attachment()?; self.branch @@ -952,8 +948,6 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, - InnerDigest: Debug, - M: Debug { self.branch .batch_maybe_push_digest(parameters, leaf_digests) @@ -986,7 +980,6 @@ where C: Configuration + ?Sized, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, { /// Builds a new [`ForkedTree`] from `leaf_digests` and `path` without checking that /// `path` is consistent with the leaves and that it is a [`CurrentPath`]. @@ -1010,8 +1003,6 @@ where M: Default + InnerMap, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, - M: Debug { #[inline] fn new(parameters: &Parameters) -> Self { @@ -1067,8 +1058,6 @@ where M: Default + InnerMap, LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default, - InnerDigest: Debug, - M: Debug { #[inline] fn leaf_digest(&self, index: usize) -> Option<&LeafDigest> { diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index fbd160a8b..c9300dd2e 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -714,8 +714,7 @@ where let indices = node.map_indices(); let mut result = Vec::new(); for (index, inner_digest) in indices.clone().zip(inner_digests) { - self.map - .set(index, inner_digest); + self.map.set(index, inner_digest); } match node.dual_parity().0 { (Parity::Left, _) => { diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index 3c922c74a..7a1d78dca 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -65,7 +65,7 @@ where leaf_digests: Vec>, /// Inner Digests - pub inner_digests: PartialInnerTree, + inner_digests: PartialInnerTree, } impl Partial @@ -278,13 +278,10 @@ where leaf_digests: Vec>, ) where LeafDigest: Default, - InnerDigest: Debug, - M: Debug { let base_inner_digests = leaf_indices.join_leaves(parameters, &leaf_digests, |node| { self.get_leaf_sibling(node) }); - println!("Inner digests: {:?}", base_inner_digests); self.inner_digests .batch_insert(parameters, leaf_indices, base_inner_digests); self.leaf_digests.extend(leaf_digests); @@ -300,10 +297,7 @@ where where F: FnOnce() -> Vec>, LeafDigest: Default, - InnerDigest: Debug, - M: Debug { - // TODO: Push without keeping unnecessary proof. let len = self.len(); let leaf_digests = leaf_digests(); let number_of_leaf_digests = leaf_digests.len(); @@ -365,8 +359,6 @@ where M: InnerMap + Default, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, - M: Debug { #[inline] fn new(parameters: &Parameters) -> Self { @@ -422,8 +414,6 @@ where M: Default + InnerMap, LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, - M: Debug { #[inline] fn leaf_digest(&self, index: usize) -> Option<&LeafDigest> { diff --git a/manta-crypto/src/merkle_tree/test/batch_insertion.rs b/manta-crypto/src/merkle_tree/test/batch_insertion.rs index 7968f40bb..1bc5e6838 100644 --- a/manta-crypto/src/merkle_tree/test/batch_insertion.rs +++ b/manta-crypto/src/merkle_tree/test/batch_insertion.rs @@ -17,35 +17,73 @@ //! Batch Insertion use crate::{ - accumulator::{Accumulator, BatchInsertion}, merkle_tree::{ - inner_tree::{InnerNode, InnerNodeRangeIter}, node::Node, partial::PartialMerkleTree, test::Test, tree::Parameters, full::FullMerkleTree + fork, full, partial, + test::Test, + tree::{Parameters, Tree}, + Leaf, }, rand::{OsRng, Rand, Sample}, }; +use core::fmt::Debug; /// Merkle Tree Height -const HEIGHT: usize = 4; +const HEIGHT: usize = 7; /// Merkle Tree Configuration -type Config = Test; +type Config = Test; -/// -#[test] -fn test_batch_insertion() { +/// Full Merkle Tree +type Full = full::Full; + +/// Partial Merkle Tree +type Partial = partial::Partial; + +/// Forked Merkle Tree +type ForkedTree = fork::ForkedTree>; + +/// Tests that batch inserting new leaves into a Merkle tree yields the same result +/// as inserting them one by one. +#[inline] +fn test_batch_insertion(f: F) +where + T: Tree + Clone + Debug + PartialEq, + Leaf: Sample, + F: FnOnce(&Parameters) -> T, +{ let mut rng = OsRng; let parameters = Parameters::::sample(Default::default(), &mut rng); - let mut tree = PartialMerkleTree::::new(parameters); + let mut tree = f(¶meters); let mut cloned_tree = tree.clone(); - let insertions = vec!["a", "b", "c", "d", "e", "f", "g", "h"].into_iter().map(String::from).collect::>(); + let number_of_insertions = rng.gen_range(1..(1 << (HEIGHT - 1))); + let mut insertions = Vec::with_capacity(number_of_insertions); + for _ in 0..number_of_insertions { + insertions.push(rng.gen()); + } for leaf in &insertions { - tree.insert(leaf); + tree.push(¶meters, leaf); } - println!("{insertions:?}"); - cloned_tree.batch_insert(&insertions); - let mut node = InnerNodeRangeIter::from_leaves::(0.into(), 7); - println!("{:?}", node.next()); - println!("{:?}", node.next()); - println!("{:?}", node.next()); - assert_eq!(tree.tree, cloned_tree.tree); -} \ No newline at end of file + cloned_tree.batch_push(¶meters, &insertions); + assert_eq!( + tree, cloned_tree, + "Individual insertions and batch insertions should yield the same results." + ); +} + +/// Runs [`test_batch_insertion`] on a [`Full`] Merkle tree. +#[test] +fn test_batch_insertion_full() { + test_batch_insertion(|parameters| Full::new(parameters)) +} + +/// Runs [`test_batch_insertion`] on a [`Partial`] Merkle tree. +#[test] +fn test_batch_insertion_partial() { + test_batch_insertion(|parameters| Partial::new(parameters)) +} + +/// Runs [`test_batch_insertion`] on a [`ForkedTree`]. +#[test] +fn test_batch_insertion_fork() { + test_batch_insertion(|parameters| ForkedTree::new(Partial::new(parameters), parameters)) +} diff --git a/manta-crypto/src/merkle_tree/tree.rs b/manta-crypto/src/merkle_tree/tree.rs index c80fc20d2..18e24dee4 100644 --- a/manta-crypto/src/merkle_tree/tree.rs +++ b/manta-crypto/src/merkle_tree/tree.rs @@ -1302,8 +1302,6 @@ where M: Default + InnerMap, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, - M: Debug { #[inline] fn rollback(&mut self) { From a0eaff60ad48c945961f9b6c51123ecd4567063b Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 18:44:03 +0200 Subject: [PATCH 40/58] test for forest --- .../src/merkle_tree/test/batch_insertion.rs | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/manta-crypto/src/merkle_tree/test/batch_insertion.rs b/manta-crypto/src/merkle_tree/test/batch_insertion.rs index 1bc5e6838..6391b77d1 100644 --- a/manta-crypto/src/merkle_tree/test/batch_insertion.rs +++ b/manta-crypto/src/merkle_tree/test/batch_insertion.rs @@ -17,8 +17,9 @@ //! Batch Insertion use crate::{ + accumulator::{Accumulator, BatchInsertion}, merkle_tree::{ - fork, full, partial, + forest, fork, full, partial, test::Test, tree::{Parameters, Tree}, Leaf, @@ -28,7 +29,7 @@ use crate::{ use core::fmt::Debug; /// Merkle Tree Height -const HEIGHT: usize = 7; +const HEIGHT: usize = 11; /// Merkle Tree Configuration type Config = Test; @@ -42,6 +43,9 @@ type Partial = partial::Partial; /// Forked Merkle Tree type ForkedTree = fork::ForkedTree>; +/// Merkle Forest Type +type Forest = forest::TreeArrayMerkleForest; + /// Tests that batch inserting new leaves into a Merkle tree yields the same result /// as inserting them one by one. #[inline] @@ -87,3 +91,25 @@ fn test_batch_insertion_partial() { fn test_batch_insertion_fork() { test_batch_insertion(|parameters| ForkedTree::new(Partial::new(parameters), parameters)) } + +/// Tests batch insertion on a Merkle forest. +#[test] +fn test_batch_insertion_forest() { + let mut rng = OsRng; + let parameters = Parameters::::sample(Default::default(), &mut rng); + let mut forest = Forest::new(parameters); + let mut cloned_forest = forest.clone(); + let number_of_insertions = rng.gen_range(1..(1 << (HEIGHT - 1))); + let mut insertions = Vec::::with_capacity(number_of_insertions); + for _ in 0..number_of_insertions { + insertions.push(rng.gen()); + } + for leaf in &insertions { + forest.insert(leaf); + } + cloned_forest.batch_insert(&insertions); + assert_eq!( + forest, cloned_forest, + "Individual insertions and batch insertions should yield the same results." + ); +} From 5278015b830ae1bea589c38d5f450b3ed4424b51 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 18:45:49 +0200 Subject: [PATCH 41/58] remove debug --- manta-crypto/src/merkle_tree/forest.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 438c0b30d..788c80ef9 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -534,7 +534,6 @@ macro_rules! impl_from_items_and_witnesses { C::Index: FixedIndex, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, { /// Builds a new [`TreeArray`] from `leaves` and `paths` without checking that /// the `paths` are consistent with the leaves and that they are @@ -567,7 +566,6 @@ macro_rules! impl_from_items_and_witnesses { Parameters: Clone, LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, { const NUMBER_OF_SUBACCUMULATORS: usize = N; @@ -603,7 +601,7 @@ macro_rules! impl_from_items_and_witnesses { Parameters: Clone, Leaf: Clone, LeafDigest: Clone + Default + PartialEq, - InnerDigest: Clone + Default + PartialEq + Debug, + InnerDigest: Clone + Default + PartialEq, { /// #[inline] @@ -773,8 +771,6 @@ where M: Default + InnerMap, LeafDigest: Clone + Default, InnerDigest: Clone + Default + PartialEq, - InnerDigest: Debug, - M: Debug, { #[inline] fn rollback(&mut self) { From 6eaa823d0c89799ed5c9fe240a811eff517db48f Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 20:16:07 +0200 Subject: [PATCH 42/58] signer updated --- manta-accounting/src/wallet/signer/functions.rs | 10 ++++++++-- manta-accounting/src/wallet/signer/mod.rs | 5 +++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 431e17ec0..c2b15426b 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -42,7 +42,9 @@ use crate::{ }; use alloc::{vec, vec::Vec}; use manta_crypto::{ - accumulator::{Accumulator, FromItemsAndWitnesses, ItemHashFunction, OptimizedAccumulator}, + accumulator::{ + Accumulator, BatchInsertion, FromItemsAndWitnesses, ItemHashFunction, OptimizedAccumulator, + }, rand::Rand, }; use manta_util::{ @@ -216,9 +218,12 @@ where let mut deposit = Vec::new(); let mut withdraw = Vec::new(); let decryption_key = parameters.derive_decryption_key(authorization_context); + let mut nonprovable_inserts = Vec::new(); for (utxo, note) in inserts { if let Some((identifier, asset)) = parameters.open_with_check(&decryption_key, &utxo, note) { + utxo_accumulator.batch_insert_nonprovable(&nonprovable_inserts); + nonprovable_inserts.clear(); insert_next_item::( authorization_context, utxo_accumulator, @@ -231,9 +236,10 @@ where rng, ); } else { - utxo_accumulator.insert_nonprovable(&item_hash::(parameters, &utxo)); + nonprovable_inserts.push(item_hash::(parameters, &utxo)); } } + utxo_accumulator.batch_insert_nonprovable(&nonprovable_inserts); assets.retain(|identifier, assets| { assets.retain(|asset| { is_asset_unspent::( diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 13fd6259f..01aea60d7 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -41,7 +41,7 @@ use alloc::{boxed::Box, vec::Vec}; use core::{convert::Infallible, fmt::Debug, hash::Hash}; use manta_crypto::{ accumulator::{ - Accumulator, ExactSizeAccumulator, FromItemsAndWitnesses, ItemHashFunction, + Accumulator, BatchInsertion, ExactSizeAccumulator, FromItemsAndWitnesses, ItemHashFunction, OptimizedAccumulator, }, rand::{CryptoRng, FromEntropy, RngCore}, @@ -867,7 +867,8 @@ pub trait Configuration: transfer::Configuration { Item = UtxoAccumulatorItem, Model = UtxoAccumulatorModel, Witness = UtxoAccumulatorWitness, - > + ExactSizeAccumulator + > + BatchInsertion + + ExactSizeAccumulator + FromItemsAndWitnesses + OptimizedAccumulator + Rollback; From 2a9212bedee88fb841e0cdb7bb1e0ba75ad53fef Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 28 Mar 2023 20:19:23 +0200 Subject: [PATCH 43/58] small check --- manta-accounting/src/wallet/signer/functions.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index c2b15426b..06217b4cd 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -222,8 +222,10 @@ where for (utxo, note) in inserts { if let Some((identifier, asset)) = parameters.open_with_check(&decryption_key, &utxo, note) { - utxo_accumulator.batch_insert_nonprovable(&nonprovable_inserts); - nonprovable_inserts.clear(); + if !nonprovable_inserts.is_empty() { + utxo_accumulator.batch_insert_nonprovable(&nonprovable_inserts); + nonprovable_inserts.clear(); + } insert_next_item::( authorization_context, utxo_accumulator, From 725d813614dd1fba0d0535b9e5c65be6adafacb9 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 13:37:14 +0200 Subject: [PATCH 44/58] batch_insert doesnt panic --- manta-accounting/src/wallet/signer/functions.rs | 4 +++- manta-crypto/src/merkle_tree/tree.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index 06217b4cd..f90506a95 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -241,7 +241,9 @@ where nonprovable_inserts.push(item_hash::(parameters, &utxo)); } } - utxo_accumulator.batch_insert_nonprovable(&nonprovable_inserts); + if !nonprovable_inserts.is_empty() { + utxo_accumulator.batch_insert_nonprovable(&nonprovable_inserts); + } assets.retain(|identifier, assets| { assets.retain(|asset| { is_asset_unspent::( diff --git a/manta-crypto/src/merkle_tree/tree.rs b/manta-crypto/src/merkle_tree/tree.rs index 18e24dee4..80f15be97 100644 --- a/manta-crypto/src/merkle_tree/tree.rs +++ b/manta-crypto/src/merkle_tree/tree.rs @@ -331,7 +331,7 @@ where F: FnOnce() -> Vec>, { self.batch_maybe_push_digest(parameters, leaf_digests) - .unwrap() + .unwrap_or(true) } /// From 5259da7c788f238ca16e3fdeb04a53e7c245cd1b Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 15:46:04 +0200 Subject: [PATCH 45/58] docs --- manta-crypto/src/accumulator.rs | 18 +++++++ manta-crypto/src/merkle_tree/forest.rs | 14 ++--- manta-crypto/src/merkle_tree/fork.rs | 7 ++- manta-crypto/src/merkle_tree/full.rs | 8 ++- manta-crypto/src/merkle_tree/inner_tree.rs | 61 ++++++++++++---------- manta-crypto/src/merkle_tree/node.rs | 15 +++--- manta-crypto/src/merkle_tree/partial.rs | 4 +- manta-crypto/src/merkle_tree/tree.rs | 38 ++++++++++++-- 8 files changed, 112 insertions(+), 53 deletions(-) diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 883901eb6..4b729cd2e 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -205,7 +205,16 @@ pub trait OptimizedAccumulator: Accumulator { /// Batch Insertion pub trait BatchInsertion: OptimizedAccumulator { + /// Inserts `items` in `self` provably, see [`insert`] for more details. + /// Returns `true` if all the insertions succeed and `false` otherwise. /// + /// # Implementation Note + /// + /// By default, this method calls [`insert`] individually for each item in `items` + /// till failure. Custom implementations of this method to improve performance + /// must return the same result as the default implementation. + /// + /// [`insert`]: Accumulator::insert #[inline] fn batch_insert<'a, I>(&mut self, items: I) -> bool where @@ -220,7 +229,16 @@ pub trait BatchInsertion: OptimizedAccumulator { true } + /// Inserts `items` in `self` nonprovably, see [`insert_nonprovable`] for more details. + /// Returns `true` if all the insertions succeed and `false` otherwise. + /// + /// # Implementation Note + /// + /// By default, this method calls [`insert_nonprovable`] individually for each item + /// in `items` till failure. Custom implementations of this method to improve performance + /// must return the same result as the default implementation. /// + /// [`insert_nonprovable`]: OptimizedAccumulator::insert_nonprovable #[inline] fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool where diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 788c80ef9..795c27949 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -603,32 +603,34 @@ macro_rules! impl_from_items_and_witnesses { LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { - /// #[inline] fn batch_insert<'a, I>(&mut self, items: I) -> bool where Self::Item: 'a, I: IntoIterator, { - let grouped_items = Self::sort_items(items.into_iter().cloned().collect()); let mut result = true; - for (index, group) in grouped_items.into_iter().enumerate() { + for (index, group) in Self::sort_items(items.into_iter().cloned().collect()) + .into_iter() + .enumerate() + { let tree = self.forest.get_mut(C::Index::from_index(index)); result &= tree.batch_push_provable(&self.parameters, &group); } result } - /// #[inline] fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool where Self::Item: 'a, I: IntoIterator, { - let grouped_items = Self::sort_items(items.into_iter().cloned().collect()); let mut result = true; - for (index, group) in grouped_items.into_iter().enumerate() { + for (index, group) in Self::sort_items(items.into_iter().cloned().collect()) + .into_iter() + .enumerate() + { let tree = self.forest.get_mut(C::Index::from_index(index)); result &= tree.batch_push(&self.parameters, &group); } diff --git a/manta-crypto/src/merkle_tree/fork.rs b/manta-crypto/src/merkle_tree/fork.rs index 3ced19328..b3232252f 100644 --- a/manta-crypto/src/merkle_tree/fork.rs +++ b/manta-crypto/src/merkle_tree/fork.rs @@ -521,7 +521,7 @@ where self.data.push(parameters, leaf) } - /// + /// Appends `leaf_digests` onto this branch. #[inline] fn batch_maybe_push_digest( &mut self, @@ -773,7 +773,10 @@ where self.branch.maybe_push_digest(parameters, leaf_digest) } + /// Appends `leaf_digests` onto this fork. /// + /// Returns `None` if this fork has been detached from its trunk. Use [`attach`](Self::attach) + /// to re-associate a trunk to this fork. #[inline] pub fn batch_maybe_push_digest( &mut self, @@ -938,7 +941,7 @@ where self.branch.maybe_push_digest(parameters, leaf_digest) } - /// + /// Appends `leaf_digests` onto this forked tree. #[inline] fn batch_maybe_push_digest( &mut self, diff --git a/manta-crypto/src/merkle_tree/full.rs b/manta-crypto/src/merkle_tree/full.rs index 866e4c9d9..cc0fbbb70 100644 --- a/manta-crypto/src/merkle_tree/full.rs +++ b/manta-crypto/src/merkle_tree/full.rs @@ -21,8 +21,8 @@ use crate::merkle_tree::{ capacity, inner_tree::{BTreeMap, InnerMap, InnerTree}, - Configuration, CurrentPath, InnerDigest, LeafDigest, MerkleTree, Node, Parameters, Path, - PathError, Root, Tree, WithProofs, + Configuration, CurrentPath, InnerDigest, LeafDigest, MerkleTree, Node, NodeRange, Parameters, + Path, PathError, Root, Tree, WithProofs, }; use alloc::vec::Vec; use core::{fmt::Debug, hash::Hash}; @@ -30,8 +30,6 @@ use core::{fmt::Debug, hash::Hash}; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; -use super::NodeRange; - /// Full Merkle Tree Type pub type FullMerkleTree> = MerkleTree>; @@ -156,7 +154,7 @@ where self.leaf_digests.push(leaf_digest); } - /// + /// Appends `leaf_digests` with indices given by `leaf_indices` into the tree. #[inline] fn batch_push_leaf_digests( &mut self, diff --git a/manta-crypto/src/merkle_tree/inner_tree.rs b/manta-crypto/src/merkle_tree/inner_tree.rs index c9300dd2e..8fd2bffd6 100644 --- a/manta-crypto/src/merkle_tree/inner_tree.rs +++ b/manta-crypto/src/merkle_tree/inner_tree.rs @@ -205,7 +205,7 @@ impl ExactSizeIterator for InnerNodeIter {} impl FusedIterator for InnerNodeIter {} -/// +/// Inner Node Range #[cfg_attr( feature = "serde", derive(Deserialize, Serialize), @@ -213,32 +213,33 @@ impl FusedIterator for InnerNodeIter {} )] #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub struct InnerNodeRange { - /// + /// Depth depth: usize, - /// + /// Node Range node_range: NodeRange, } impl InnerNodeRange { - /// + /// Builds a new [`InnerNodeRange`] from `depth` and `node_range` without + /// checking that the `node_range` fits in a binary Merkle tree at + /// the given `depth`. #[inline] const fn new_unchecked(depth: usize, node_range: NodeRange) -> Self { Self { depth, node_range } } - /// + /// Builds a new [`InnerNodeRange`] from `depth` and `node_range`. #[inline] pub const fn new(depth: usize, node_range: NodeRange) -> Option { - let last_node = node_range.last_node(); - if last_node.0 >= (1 << (depth + 1)) { + if node_range.last_node().0 >= (1 << (depth + 1)) { None } else { Some(Self::new_unchecked(depth, node_range)) } } - /// + /// Builds a new [`InnerNodeRange`] from `leaf_index` and `extra_leaves`. #[inline] pub fn from_leaves(leaf_index: Node, extra_leaves: usize) -> Option where @@ -254,13 +255,14 @@ impl InnerNodeRange { .and_then(|inner_node_range| inner_node_range.parents()) } - /// + /// Returns the [`DualParity`] of `self`. #[inline] pub const fn dual_parity(&self) -> DualParity { self.node_range.dual_parity() } - /// + /// Returns the [`InnerNodeRange`] consisting of the parents + /// of the [`InnerNode`]s in `self`. #[inline] pub const fn parents(&self) -> Option { match self.depth.checked_sub(1) { @@ -269,7 +271,7 @@ impl InnerNodeRange { } } - /// Converts `self` into its parent, if the parent exists, returning the parent [`InnerNode`]. + /// Converts `self` into its parents, if the parents exist. #[inline] pub fn into_parents(&mut self) -> Option { match self.parents() { @@ -281,19 +283,19 @@ impl InnerNodeRange { } } - /// + /// Returns the depth of `self`. #[inline] - pub const fn depth(&self) -> usize { - self.depth + pub const fn depth(&self) -> &usize { + &self.depth } - /// + /// Returns the [`NodeRange`] of `self`. #[inline] - pub const fn node_range(&self) -> NodeRange { - self.node_range + pub const fn node_range(&self) -> &NodeRange { + &self.node_range } - /// + /// Returns the starting [`InnerNode`] of `self`. #[inline] pub const fn starting_inner_node(&self) -> InnerNode { InnerNode { @@ -302,7 +304,7 @@ impl InnerNodeRange { } } - /// + /// Returns the last [`InnerNode`] of `self`. #[inline] pub const fn last_inner_node(&self) -> InnerNode { InnerNode { @@ -311,14 +313,14 @@ impl InnerNodeRange { } } - /// + /// Computes an [`InnerMap`] range of indices for the coordinates represented by `self`. #[inline] pub fn map_indices(&self) -> Range { self.starting_inner_node().map_index()..self.last_inner_node().map_index() + 1 } } -/// InnerNodeRangeIter +/// Inner Node Range Iterator #[cfg_attr( feature = "serde", derive(Deserialize, Serialize), @@ -326,7 +328,7 @@ impl InnerNodeRange { )] #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub struct InnerNodeRangeIter { - /// + /// Inner Node Range node_range: Option, } @@ -337,7 +339,7 @@ impl InnerNodeRangeIter { Self { node_range } } - /// Builds a new [`InnerNodeIter`] iterator over the parents of `leaf_index`. + /// Builds a new [`InnerNodeRangeIter`] iterator over the parents of `leaf_index`. #[inline] pub fn from_leaves(leaf_index: Node, extra_leaves: usize) -> Self where @@ -703,7 +705,9 @@ where parameters.join(lhs, rhs) } - /// + /// Inserts `inner_digests` into the tree at `node`. Computes the inner hashes of `inner_digests` + /// pairwise and in order. If the first (last) element has a right (left) parity, it will be hashed + /// with its sibling if it is stored in the tree or with the default value otherwise. #[inline] fn insert_range_and_join( &mut self, @@ -751,7 +755,8 @@ where }) } - /// + /// Computes the new root of the tree after inserting `base` which corresponds to the leaves + /// at `leaf_indices`. #[inline] fn compute_root_from_leaves( &mut self, @@ -768,7 +773,8 @@ where .expect("The final vector consists of the root of the Merkle tree.") } - /// + /// Inserts the `base` inner digests corresponding to the leaves at `leaf_indices` + /// into the tree. #[inline] pub fn batch_insert( &mut self, @@ -997,7 +1003,8 @@ where self.inner_tree.insert(parameters, leaf_index, base); } - /// + /// Inserts the `base` inner digests corresponding to the leaves at `leaf_indices` + /// into the tree. #[inline] pub fn batch_insert( &mut self, diff --git a/manta-crypto/src/merkle_tree/node.rs b/manta-crypto/src/merkle_tree/node.rs index 995e42304..41c351422 100644 --- a/manta-crypto/src/merkle_tree/node.rs +++ b/manta-crypto/src/merkle_tree/node.rs @@ -451,13 +451,13 @@ impl FusedIterator for NodeParents {} pub struct DualParity(pub (Parity, Parity)); impl DualParity { - /// + /// Returns the starting [`Parity`] of `self`. #[inline] pub const fn starting_parity(&self) -> Parity { self.0 .0 } - /// + /// Returns the final [`Parity`] of `self`. #[inline] pub const fn final_parity(&self) -> Parity { self.0 .1 @@ -480,7 +480,7 @@ pub struct NodeRange { } impl NodeRange { - /// + /// Returns the [`DualParity`] of `self`. #[inline] pub const fn dual_parity(&self) -> DualParity { let starting_node_parity = self.node.parity(); @@ -491,7 +491,8 @@ impl NodeRange { DualParity((starting_node_parity, final_node_parity)) } - /// + /// Returns the [`NodeRange`] consisting of the parents of the + /// [`Node`]s in `self`. #[inline] pub const fn parents(&self) -> Self { let extra_nodes = match self.dual_parity().0 { @@ -505,13 +506,15 @@ impl NodeRange { } } - /// + /// Returns the last [`Node`] in `self`. #[inline] pub const fn last_node(&self) -> Node { Node(self.node.0 + self.extra_nodes) } - /// + /// Computes the inner hashes of `leaves` pairwise and in order. If the first (last) element has a + /// right (left) [`Parity`], it will be hashed with the output of `get_leaves` or with the default + /// value if `get_leaves` returns `None`. #[inline] pub fn join_leaves<'a, C, F>( &self, diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index 7a1d78dca..7bdf2f06f 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -269,7 +269,7 @@ where self.leaf_digests.push(leaf_digest); } - /// + /// Appends `leaf_digests` with indices given by `leaf_indices` into the tree. #[inline] fn batch_push_leaf_digests( &mut self, @@ -287,7 +287,7 @@ where self.leaf_digests.extend(leaf_digests); } - /// + /// Appends `leaf_digests` to the tree using `parameters`. #[inline] pub fn batch_maybe_push_digest( &mut self, diff --git a/manta-crypto/src/merkle_tree/tree.rs b/manta-crypto/src/merkle_tree/tree.rs index 80f15be97..c27892b14 100644 --- a/manta-crypto/src/merkle_tree/tree.rs +++ b/manta-crypto/src/merkle_tree/tree.rs @@ -302,7 +302,16 @@ where self.push_digest(parameters, || parameters.digest(leaf)) } + /// Inserts `leaf_digests` in `self`, see [`maybe_push_digest`] for more details. + /// Returns `true` if all the insertions succeed and `false` otherwise. /// + /// # Implementation Note + /// + /// By default, this method calls [`maybe_push_digest`] individually for each leaf digest + /// in `leaf_digests` till failure. Custom implementations of this method to improve performance + /// must return the same result as the default implementation. + /// + /// [`maybe_push_digest`]: Tree::maybe_push_digest #[inline] fn batch_maybe_push_digest( &mut self, @@ -324,7 +333,8 @@ where Some(true) } - /// + /// Inserts `leaf_digests` in `self`. Returns `true` if all the insertions + /// succeed or if there were no `leaf_digests` and `false` otherwise. #[inline] fn batch_push_digest(&mut self, parameters: &Parameters, leaf_digests: F) -> bool where @@ -334,7 +344,8 @@ where .unwrap_or(true) } - /// + /// Inserts the digests of `leaves` at the next available leaf node of the tree, returning + /// `false` if the leaves could not be inserted because the tree has exhausted its capacity. #[inline] fn batch_push<'a, I>(&mut self, parameters: &Parameters, leaves: I) -> bool where @@ -499,7 +510,16 @@ where self.push_provable_digest(parameters, move || parameters.digest(leaf)) } + /// Inserts `leaf_digests` provably in `self`, see [`maybe_push_provable_digest`] for more details. + /// Returns `true` if all the insertions succeed and `false` otherwise. + /// + /// # Implementation Note + /// + /// By default, this method calls [`maybe_push_provable_digest`] individually for each leaf digest + /// in `leaf_digests` till failure. Custom implementations of this method to improve performance + /// must return the same result as the default implementation. /// + /// [`maybe_push_provable_digest`]: WithProofs::maybe_push_provable_digest #[inline] fn batch_maybe_push_provable_digest( &mut self, @@ -521,17 +541,19 @@ where Some(true) } - /// + /// Inserts `leaf_digests` provably in `self`. Returns `true` if all the insertions + /// succeed or if there were no `leaf_digests` and `false` otherwise. #[inline] fn batch_push_provable_digest(&mut self, parameters: &Parameters, leaf_digests: F) -> bool where F: FnOnce() -> Vec>, { self.batch_maybe_push_provable_digest(parameters, leaf_digests) - .unwrap() + .unwrap_or(true) } - /// + /// Inserts the digests of `leaves` provably at the next available leaf node of the tree, returning + /// `false` if the leaves could not be inserted because the tree has exhausted its capacity. #[inline] fn batch_push_provable<'a, I>(&mut self, parameters: &Parameters, leaves: I) -> bool where @@ -1008,7 +1030,10 @@ where self.tree.push(&self.parameters, leaf) } + /// Inserts `leaves` at the next avaiable leaf node of the tree, returning `true` if all + /// leaf insertions succeed and `false` otherwise. /// + /// See [`Tree::batch_push`] for more. #[inline] pub fn batch_push<'a, I>(&mut self, leaves: I) -> bool where @@ -1100,7 +1125,10 @@ where self.tree.push_provable(&self.parameters, leaf) } + /// Inserts `leaves` provably at the next avaiable leaf node of the tree, returning `true` if all + /// provable leaf insertions succeed and `false` otherwise. /// + /// See [`WithProofs::batch_push_provable`] for more. #[inline] pub fn batch_push_provable<'a, I>(&mut self, leaves: I) -> bool where From 476fcba93c4bd1533ed9d04d1beb4ddac75f679b Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 15:56:48 +0200 Subject: [PATCH 46/58] docs --- manta-crypto/src/merkle_tree/forest.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index 795c27949..e1f205c51 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -603,6 +603,11 @@ macro_rules! impl_from_items_and_witnesses { LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { + /// # Note + /// + /// This implementation does not yield the same result as individually inserting + /// each item in `items` with [`insert`](Accumulator::insert) till failure. + /// However, this property holds at the tree level. #[inline] fn batch_insert<'a, I>(&mut self, items: I) -> bool where @@ -620,6 +625,11 @@ macro_rules! impl_from_items_and_witnesses { result } + /// # Note + /// + /// This implementation does not yield the same result as individually inserting + /// each item in `items` with [`insert_nonprovable`](OptimizedAccumulator::insert_nonprovable) + /// till failure. However, this property holds at the tree level. #[inline] fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool where From b11308cb4892f6e7c468530eabb4d43226f66777 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 16:47:06 +0200 Subject: [PATCH 47/58] batch insertion till full capacity --- manta-crypto/src/merkle_tree/full.rs | 40 +++++++++++++++++-------- manta-crypto/src/merkle_tree/partial.rs | 39 ++++++++++++++++-------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/manta-crypto/src/merkle_tree/full.rs b/manta-crypto/src/merkle_tree/full.rs index cc0fbbb70..dbe5183d5 100644 --- a/manta-crypto/src/merkle_tree/full.rs +++ b/manta-crypto/src/merkle_tree/full.rs @@ -26,6 +26,7 @@ use crate::merkle_tree::{ }; use alloc::vec::Vec; use core::{fmt::Debug, hash::Hash}; +use manta_util::vec::VecExt; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; @@ -237,24 +238,37 @@ where where F: FnOnce() -> Vec>, { - let len = self.len(); let leaf_digests = leaf_digests(); - let number_of_leaf_digests = leaf_digests.len(); if leaf_digests.is_empty() { return None; } - if len + number_of_leaf_digests > capacity::() { - return Some(false); + let len = self.len(); + let number_of_leaf_digests = leaf_digests.len(); + let capacity = capacity::(); + if len + number_of_leaf_digests > capacity { + let max_number_of_insertions = capacity - len; + if max_number_of_insertions != 0 { + self.batch_push_leaf_digests( + parameters, + NodeRange { + node: Node(len), + extra_nodes: max_number_of_insertions - 1, + }, + leaf_digests[..max_number_of_insertions].to_vec(), + ); + } + Some(false) + } else { + self.batch_push_leaf_digests( + parameters, + NodeRange { + node: Node(len), + extra_nodes: number_of_leaf_digests - 1, + }, + leaf_digests, + ); + Some(true) } - self.batch_push_leaf_digests( - parameters, - NodeRange { - node: Node(len), - extra_nodes: number_of_leaf_digests - 1, - }, - leaf_digests, - ); - Some(true) } } diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index 7bdf2f06f..775460884 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -298,24 +298,37 @@ where F: FnOnce() -> Vec>, LeafDigest: Default, { - let len = self.len(); let leaf_digests = leaf_digests(); - let number_of_leaf_digests = leaf_digests.len(); if leaf_digests.is_empty() { return None; } - if len + number_of_leaf_digests > capacity::() { - return Some(false); + let len = self.len(); + let number_of_leaf_digests = leaf_digests.len(); + let capacity = capacity::(); + if len + number_of_leaf_digests > capacity { + let max_number_of_insertions = capacity - len; + if max_number_of_insertions != 0 { + self.batch_push_leaf_digests( + parameters, + NodeRange { + node: Node(len), + extra_nodes: max_number_of_insertions - 1, + }, + leaf_digests[..max_number_of_insertions].to_vec(), + ); + } + Some(false) + } else { + self.batch_push_leaf_digests( + parameters, + NodeRange { + node: Node(len), + extra_nodes: number_of_leaf_digests - 1, + }, + leaf_digests, + ); + Some(true) } - self.batch_push_leaf_digests( - parameters, - NodeRange { - node: Node(len), - extra_nodes: number_of_leaf_digests - 1, - }, - leaf_digests, - ); - Some(true) } /// Appends a `leaf` to the tree using `parameters`. From d5bbb4aceefbe36a454f316ac03efa9237f56280 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 16:49:41 +0200 Subject: [PATCH 48/58] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0055f4bbf..10706fb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added +- [\#330](https://github.com/Manta-Network/manta-rs/pull/330) Merkle tree batch insertions. - [\#329](https://github.com/Manta-Network/manta-rs/pull/329) Signer initial synchronization method. - [\#328](https://github.com/Manta-Network/manta-rs/pull/328) Expose reset wallet method. From eecf9ff6812e67ce8e898fb8f554d6d9140b5427 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 17:47:14 +0200 Subject: [PATCH 49/58] truncate --- manta-crypto/src/merkle_tree/full.rs | 6 +++--- manta-crypto/src/merkle_tree/partial.rs | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/manta-crypto/src/merkle_tree/full.rs b/manta-crypto/src/merkle_tree/full.rs index dbe5183d5..6eb168847 100644 --- a/manta-crypto/src/merkle_tree/full.rs +++ b/manta-crypto/src/merkle_tree/full.rs @@ -26,7 +26,6 @@ use crate::merkle_tree::{ }; use alloc::vec::Vec; use core::{fmt::Debug, hash::Hash}; -use manta_util::vec::VecExt; #[cfg(feature = "serde")] use manta_util::serde::{Deserialize, Serialize}; @@ -238,7 +237,7 @@ where where F: FnOnce() -> Vec>, { - let leaf_digests = leaf_digests(); + let mut leaf_digests = leaf_digests(); if leaf_digests.is_empty() { return None; } @@ -248,13 +247,14 @@ where if len + number_of_leaf_digests > capacity { let max_number_of_insertions = capacity - len; if max_number_of_insertions != 0 { + leaf_digests.truncate(max_number_of_insertions); self.batch_push_leaf_digests( parameters, NodeRange { node: Node(len), extra_nodes: max_number_of_insertions - 1, }, - leaf_digests[..max_number_of_insertions].to_vec(), + leaf_digests, ); } Some(false) diff --git a/manta-crypto/src/merkle_tree/partial.rs b/manta-crypto/src/merkle_tree/partial.rs index 775460884..7135effc8 100644 --- a/manta-crypto/src/merkle_tree/partial.rs +++ b/manta-crypto/src/merkle_tree/partial.rs @@ -298,7 +298,7 @@ where F: FnOnce() -> Vec>, LeafDigest: Default, { - let leaf_digests = leaf_digests(); + let mut leaf_digests = leaf_digests(); if leaf_digests.is_empty() { return None; } @@ -308,13 +308,14 @@ where if len + number_of_leaf_digests > capacity { let max_number_of_insertions = capacity - len; if max_number_of_insertions != 0 { + leaf_digests.truncate(max_number_of_insertions); self.batch_push_leaf_digests( parameters, NodeRange { node: Node(len), extra_nodes: max_number_of_insertions - 1, }, - leaf_digests[..max_number_of_insertions].to_vec(), + leaf_digests, ); } Some(false) From d4c87182f8188dde36a1dbd7e009681fd60f4580 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 29 Mar 2023 20:26:33 +0200 Subject: [PATCH 50/58] expose auth context --- manta-accounting/src/wallet/signer/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 01aea60d7..b7be441f8 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -1089,14 +1089,14 @@ where /// Returns the [`AccountTable`]. #[inline] - pub fn accounts(&self) -> &Option> { - &self.accounts + pub fn accounts(&self) -> Option<&AccountTable> { + self.accounts.as_ref() } /// Returns the [`AuthorizationContext`]. #[inline] - pub fn authorization_context(&self) -> &Option> { - &self.authorization_context + pub fn authorization_context(&self) -> Option<&AuthorizationContext> { + self.authorization_context.as_ref() } /// Returns the default account for `self`. @@ -1360,6 +1360,12 @@ where )) } + /// Returns the [`AuthorizationContext`] corresponding to `self`. + #[inline] + pub fn authorization_context(&self) -> Option<&AuthorizationContext> { + self.state.authorization_context.as_ref() + } + /// Returns the associated [`TransactionData`] of `post`, namely the [`Asset`] and the /// [`Identifier`]. Returns `None` if `post` has an invalid shape, or if `self` doesn't own the /// underlying assets in `post`. From d7977e145745e28334d62a0023eb0555aaa87035 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 3 Apr 2023 20:50:45 +0200 Subject: [PATCH 51/58] comments addressed --- manta-crypto/src/accumulator.rs | 4 +- manta-crypto/src/merkle_tree/node.rs | 13 +++-- .../src/merkle_tree/test/batch_insertion.rs | 57 ++++++++++--------- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 4b729cd2e..481c3b46a 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -221,7 +221,7 @@ pub trait BatchInsertion: OptimizedAccumulator { Self::Item: 'a, I: IntoIterator, { - for item in items.into_iter() { + for item in items { if !self.insert(item) { return false; } @@ -245,7 +245,7 @@ pub trait BatchInsertion: OptimizedAccumulator { Self::Item: 'a, I: IntoIterator, { - for item in items.into_iter() { + for item in items { if !self.insert_nonprovable(item) { return false; } diff --git a/manta-crypto/src/merkle_tree/node.rs b/manta-crypto/src/merkle_tree/node.rs index 41c351422..4c75e1df5 100644 --- a/manta-crypto/src/merkle_tree/node.rs +++ b/manta-crypto/src/merkle_tree/node.rs @@ -160,7 +160,7 @@ impl Default for Parity { derive(Deserialize, Serialize), serde(crate = "manta_util::serde", deny_unknown_fields) )] -#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub struct Node( /// Level-wise Index to a node in a Binary Tree pub Idx, @@ -442,6 +442,11 @@ impl Iterator for NodeParents { impl FusedIterator for NodeParents {} /// Dual Parity +/// +/// # Note +/// +/// Given a [`NodeRange`], this struct describes the parity of its left-most +/// and right-most [`Node`]s. #[cfg_attr( feature = "serde", derive(Deserialize, Serialize), @@ -454,13 +459,13 @@ impl DualParity { /// Returns the starting [`Parity`] of `self`. #[inline] pub const fn starting_parity(&self) -> Parity { - self.0 .0 + (self.0).0 } /// Returns the final [`Parity`] of `self`. #[inline] pub const fn final_parity(&self) -> Parity { - self.0 .1 + (self.0).1 } } @@ -470,7 +475,7 @@ impl DualParity { derive(Deserialize, Serialize), serde(crate = "manta_util::serde", deny_unknown_fields) )] -#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] pub struct NodeRange { /// Starting Node pub node: Node, diff --git a/manta-crypto/src/merkle_tree/test/batch_insertion.rs b/manta-crypto/src/merkle_tree/test/batch_insertion.rs index 6391b77d1..3fcb1e19d 100644 --- a/manta-crypto/src/merkle_tree/test/batch_insertion.rs +++ b/manta-crypto/src/merkle_tree/test/batch_insertion.rs @@ -49,25 +49,26 @@ type Forest = forest::TreeArrayMerkleForest; /// Tests that batch inserting new leaves into a Merkle tree yields the same result /// as inserting them one by one. #[inline] -fn test_batch_insertion(f: F) +fn test_batch_insertion(f: F, mut insert: G, mut batch_insert: H) where - T: Tree + Clone + Debug + PartialEq, + T: Clone + Debug + PartialEq, Leaf: Sample, F: FnOnce(&Parameters) -> T, + G: FnMut(&mut T, &Parameters, &Leaf) -> bool, + H: FnMut(&mut T, &Parameters, &[Leaf]) -> bool, { let mut rng = OsRng; let parameters = Parameters::::sample(Default::default(), &mut rng); let mut tree = f(¶meters); let mut cloned_tree = tree.clone(); let number_of_insertions = rng.gen_range(1..(1 << (HEIGHT - 1))); - let mut insertions = Vec::with_capacity(number_of_insertions); - for _ in 0..number_of_insertions { - insertions.push(rng.gen()); - } + let insertions = (0..number_of_insertions) + .map(|_| rng.gen()) + .collect::>(); for leaf in &insertions { - tree.push(¶meters, leaf); + insert(&mut tree, ¶meters, leaf); } - cloned_tree.batch_push(¶meters, &insertions); + batch_insert(&mut cloned_tree, ¶meters, &insertions); assert_eq!( tree, cloned_tree, "Individual insertions and batch insertions should yield the same results." @@ -77,39 +78,39 @@ where /// Runs [`test_batch_insertion`] on a [`Full`] Merkle tree. #[test] fn test_batch_insertion_full() { - test_batch_insertion(|parameters| Full::new(parameters)) + test_batch_insertion( + |parameters| Full::new(parameters), + |tree, parameters, leaf| tree.push(parameters, leaf), + |tree, parameters, leaves| tree.batch_push(parameters, leaves), + ) } /// Runs [`test_batch_insertion`] on a [`Partial`] Merkle tree. #[test] fn test_batch_insertion_partial() { - test_batch_insertion(|parameters| Partial::new(parameters)) + test_batch_insertion( + |parameters| Partial::new(parameters), + |tree, parameters, leaf| tree.push(parameters, leaf), + |tree, parameters, leaves| tree.batch_push(parameters, leaves), + ) } /// Runs [`test_batch_insertion`] on a [`ForkedTree`]. #[test] fn test_batch_insertion_fork() { - test_batch_insertion(|parameters| ForkedTree::new(Partial::new(parameters), parameters)) + test_batch_insertion( + |parameters| ForkedTree::new(Partial::new(parameters), parameters), + |tree, parameters, leaf| tree.push(parameters, leaf), + |tree, parameters, leaves| tree.batch_push(parameters, leaves), + ) } /// Tests batch insertion on a Merkle forest. #[test] fn test_batch_insertion_forest() { - let mut rng = OsRng; - let parameters = Parameters::::sample(Default::default(), &mut rng); - let mut forest = Forest::new(parameters); - let mut cloned_forest = forest.clone(); - let number_of_insertions = rng.gen_range(1..(1 << (HEIGHT - 1))); - let mut insertions = Vec::::with_capacity(number_of_insertions); - for _ in 0..number_of_insertions { - insertions.push(rng.gen()); - } - for leaf in &insertions { - forest.insert(leaf); - } - cloned_forest.batch_insert(&insertions); - assert_eq!( - forest, cloned_forest, - "Individual insertions and batch insertions should yield the same results." - ); + test_batch_insertion( + |parameters| Forest::new(parameters.clone()), + |forest, _, leaf| forest.insert(leaf), + |forest, _, leaves| forest.batch_insert(leaves), + ) } From b12df285360829c887c28d761ffa05e4fbaacf37 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 3 Apr 2023 21:04:17 +0200 Subject: [PATCH 52/58] try_load --- manta-accounting/src/wallet/signer/mod.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index b7be441f8..a72dbdd37 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -1075,6 +1075,19 @@ where self.accounts = None } + /// Tries to load `authorization_context_option` to `self`. + #[inline] + pub fn try_load_authorization_context( + &mut self, + authorization_context_option: Option>, + ) -> bool { + if let Some(authorization_context) = authorization_context_option { + self.load_authorization_context(authorization_context); + return true; + } + false + } + /// Loads `authorization_context` to `self`. #[inline] pub fn load_authorization_context(&mut self, authorization_context: AuthorizationContext) { @@ -1233,6 +1246,16 @@ where self.state.drop_accounts() } + /// Tries to load `authorization_context_option` to `self`. + #[inline] + pub fn try_load_authorization_context( + &mut self, + authorization_context_option: Option>, + ) -> bool { + self.state + .try_load_authorization_context(authorization_context_option) + } + /// Loads `authorization_context` to `self`. #[inline] pub fn load_authorization_context(&mut self, authorization_context: AuthorizationContext) { From eb3b2f39b7c43e7ac2f47847f15130b45c09d9e6 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Mon, 3 Apr 2023 22:34:37 +0200 Subject: [PATCH 53/58] clippy --- manta-crypto/src/merkle_tree/test/batch_insertion.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manta-crypto/src/merkle_tree/test/batch_insertion.rs b/manta-crypto/src/merkle_tree/test/batch_insertion.rs index 3fcb1e19d..55aa8d077 100644 --- a/manta-crypto/src/merkle_tree/test/batch_insertion.rs +++ b/manta-crypto/src/merkle_tree/test/batch_insertion.rs @@ -109,7 +109,7 @@ fn test_batch_insertion_fork() { #[test] fn test_batch_insertion_forest() { test_batch_insertion( - |parameters| Forest::new(parameters.clone()), + |parameters| Forest::new(*parameters), |forest, _, leaf| forest.insert(leaf), |forest, _, leaves| forest.batch_insert(leaves), ) From e43516e72d01b476b3de5e234f9244d419fc3499 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 4 Apr 2023 16:11:26 +0200 Subject: [PATCH 54/58] comment addressed --- manta-crypto/src/accumulator.rs | 4 ++-- manta-crypto/src/merkle_tree/forest.rs | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/manta-crypto/src/accumulator.rs b/manta-crypto/src/accumulator.rs index 481c3b46a..f91b1933f 100644 --- a/manta-crypto/src/accumulator.rs +++ b/manta-crypto/src/accumulator.rs @@ -212,7 +212,7 @@ pub trait BatchInsertion: OptimizedAccumulator { /// /// By default, this method calls [`insert`] individually for each item in `items` /// till failure. Custom implementations of this method to improve performance - /// must return the same result as the default implementation. + /// may insert a subset of `items` in case of failure. /// /// [`insert`]: Accumulator::insert #[inline] @@ -236,7 +236,7 @@ pub trait BatchInsertion: OptimizedAccumulator { /// /// By default, this method calls [`insert_nonprovable`] individually for each item /// in `items` till failure. Custom implementations of this method to improve performance - /// must return the same result as the default implementation. + /// may insert a subset of `items` in case of failure. /// /// [`insert_nonprovable`]: OptimizedAccumulator::insert_nonprovable #[inline] diff --git a/manta-crypto/src/merkle_tree/forest.rs b/manta-crypto/src/merkle_tree/forest.rs index e1f205c51..795c27949 100644 --- a/manta-crypto/src/merkle_tree/forest.rs +++ b/manta-crypto/src/merkle_tree/forest.rs @@ -603,11 +603,6 @@ macro_rules! impl_from_items_and_witnesses { LeafDigest: Clone + Default + PartialEq, InnerDigest: Clone + Default + PartialEq, { - /// # Note - /// - /// This implementation does not yield the same result as individually inserting - /// each item in `items` with [`insert`](Accumulator::insert) till failure. - /// However, this property holds at the tree level. #[inline] fn batch_insert<'a, I>(&mut self, items: I) -> bool where @@ -625,11 +620,6 @@ macro_rules! impl_from_items_and_witnesses { result } - /// # Note - /// - /// This implementation does not yield the same result as individually inserting - /// each item in `items` with [`insert_nonprovable`](OptimizedAccumulator::insert_nonprovable) - /// till failure. However, this property holds at the tree level. #[inline] fn batch_insert_nonprovable<'a, I>(&mut self, items: I) -> bool where From b167040040613f0d533768b9bdbcfa450449c181 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Tue, 4 Apr 2023 21:18:21 +0200 Subject: [PATCH 55/58] sbt signer sync --- .../src/wallet/signer/functions.rs | 170 ++++++++++++++---- manta-accounting/src/wallet/signer/mod.rs | 91 +++++++--- manta-pay/src/signer/base.rs | 18 +- manta-pay/src/signer/client/http.rs | 8 + manta-pay/src/signer/client/websocket.rs | 8 + 5 files changed, 226 insertions(+), 69 deletions(-) diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index f90506a95..a0be64779 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -129,7 +129,6 @@ fn insert_next_item( utxo_accumulator: &mut C::UtxoAccumulator, assets: &mut C::AssetMap, parameters: &Parameters, - utxo: Utxo, identified_asset: IdentifiedAsset, nullifiers: &mut Vec>, deposit: &mut Vec>, @@ -138,26 +137,24 @@ fn insert_next_item( C: Configuration, { let IdentifiedAsset:: { identifier, asset } = identified_asset; - let (_, computed_utxo, nullifier) = parameters.derive_spend( + let (_, utxo, nullifier) = parameters.derive_spend( authorization_context, identifier.clone(), asset.clone(), rng, ); - if computed_utxo.is_related(&utxo) { - if let Some(index) = nullifiers - .iter() - .position(move |n| n.is_related(&nullifier)) - { - nullifiers.remove(index); - } else { - utxo_accumulator.insert(&item_hash::(parameters, &utxo)); - if !asset.is_zero() { - deposit.push(asset.clone()); - } - assets.insert(identifier, asset); - return; + if let Some(index) = nullifiers + .iter() + .position(move |n| n.is_related(&nullifier)) + { + nullifiers.remove(index); + } else { + utxo_accumulator.insert(&item_hash::(parameters, &utxo)); + if !asset.is_zero() { + deposit.push(asset.clone()); } + assets.insert(identifier, asset); + return; } utxo_accumulator.insert_nonprovable(&item_hash::(parameters, &utxo)); } @@ -231,7 +228,6 @@ where utxo_accumulator, assets, parameters, - utxo, transfer::utxo::IdentifiedAsset::new(identifier, asset), &mut nullifiers, &mut deposit, @@ -275,6 +271,53 @@ where } } +/// Updates the internal ledger state, returning the new asset distribution. +#[allow(clippy::too_many_arguments)] +#[inline] +fn sbt_sync_with( + authorization_context: &mut AuthorizationContext, + assets: &mut C::AssetMap, + checkpoint: &mut C::Checkpoint, + parameters: &Parameters, + inserts: I, + utxo_count: Vec, + nullifier_count: usize, + is_partial: bool, +) -> SyncResponse +where + C: Configuration, + I: Iterator, Note)>, +{ + let mut deposit = Vec::new(); + let decryption_key = parameters.derive_decryption_key(authorization_context); + for (utxo, note) in inserts { + if let Some((identifier, asset)) = parameters.open_with_check(&decryption_key, &utxo, note) + { + if !asset.is_zero() { + deposit.push(asset.clone()); + } + assets.insert(identifier, asset); + } + } + checkpoint.update_from_nullifiers(nullifier_count); + checkpoint.update_from_utxo_count(utxo_count); + SyncResponse { + checkpoint: checkpoint.clone(), + balance_update: if is_partial { + // TODO: Whenever we are doing a full update, don't even build the `deposit` and + // `withdraw` vectors, since we won't be needing them. + BalanceUpdate::Partial { + deposit, + withdraw: Default::default(), + } + } else { + BalanceUpdate::Full { + assets: assets.assets().into(), + } + }, + } +} + /// Builds the [`PreSender`] associated to `identifier` and `asset`. #[inline] fn build_pre_sender( @@ -619,17 +662,14 @@ where account.address(¶meters.parameters) } -/// Updates `assets`, `checkpoint` and `utxo_accumulator`, returning the new asset distribution. +/// Checks that the origin checkpoint in `request` is less or equal than `checkpoint`. +/// If it is strictly less, it prunes the data in `request` accordingly. #[inline] -pub fn sync( +fn prune_sync_request( parameters: &SignerParameters, - authorization_context: &mut AuthorizationContext, - assets: &mut C::AssetMap, - checkpoint: &mut C::Checkpoint, - utxo_accumulator: &mut C::UtxoAccumulator, + checkpoint: &C::Checkpoint, mut request: SyncRequest, - rng: &mut C::Rng, -) -> Result, SyncError> +) -> Result<(bool, SyncData), SyncError> where C: Configuration, { @@ -647,24 +687,76 @@ where parameters.parameters.utxo_accumulator_item_hash(), checkpoint, ); - let SyncData { + Ok((has_pruned, request.data)) + } +} + +/// +#[inline] +pub fn sbt_sync( + parameters: &SignerParameters, + authorization_context: &mut AuthorizationContext, + assets: &mut C::AssetMap, + checkpoint: &mut C::Checkpoint, + request: SyncRequest, +) -> Result, SyncError> +where + C: Configuration, +{ + let utxo_count = request.utxo_count(¶meters.parameters); + let ( + has_pruned, + SyncData { utxo_note_data, nullifier_data, - } = request.data; - let response = sync_with::( - authorization_context, - assets, - checkpoint, - utxo_accumulator, - ¶meters.parameters, - utxo_note_data.into_iter(), + }, + ) = prune_sync_request(parameters, checkpoint, request)?; + Ok(sbt_sync_with( + authorization_context, + assets, + checkpoint, + ¶meters.parameters, + utxo_note_data.into_iter(), + utxo_count, + nullifier_data.len(), + !has_pruned, + )) +} + +/// Updates `assets`, `checkpoint` and `utxo_accumulator`, returning the new asset distribution. +#[inline] +pub fn sync( + parameters: &SignerParameters, + authorization_context: &mut AuthorizationContext, + assets: &mut C::AssetMap, + checkpoint: &mut C::Checkpoint, + utxo_accumulator: &mut C::UtxoAccumulator, + request: SyncRequest, + rng: &mut C::Rng, +) -> Result, SyncError> +where + C: Configuration, +{ + let ( + has_pruned, + SyncData { + utxo_note_data, nullifier_data, - !has_pruned, - rng, - ); - utxo_accumulator.commit(); - Ok(response) - } + }, + ) = prune_sync_request(parameters, checkpoint, request)?; + let response = sync_with::( + authorization_context, + assets, + checkpoint, + utxo_accumulator, + ¶meters.parameters, + utxo_note_data.into_iter(), + nullifier_data, + !has_pruned, + rng, + ); + utxo_accumulator.commit(); + Ok(response) } /// Signs a withdraw transaction for `asset` sent to `address`. diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index a72dbdd37..76605370c 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -79,6 +79,12 @@ where request: SyncRequest, ) -> LocalBoxFutureResult, Self::Error>; + /// + fn sbt_sync( + &mut self, + request: SyncRequest, + ) -> LocalBoxFutureResult, Self::Error>; + /// Performs the initial synchronization of a new signer with the ledger data. /// /// # Implementation Note @@ -235,14 +241,11 @@ where impl InitialSyncRequest where - C: transfer::Configuration, + C: Configuration, { /// Builds a new [`InitialSyncRequest`] from `parameters` and `data`. #[inline] - pub fn from_initial_sync_data(parameters: &Parameters, data: InitialSyncData) -> Self - where - C: Configuration, - { + pub fn from_initial_sync_data(parameters: &Parameters, data: InitialSyncData) -> Self { Self { utxo_data: C::UtxoAccumulator::sort_items( data.utxo_data @@ -255,29 +258,25 @@ where } } - /// Extends `self` with `parameters` and `data`. + /// #[inline] - pub fn extend_with_data(&mut self, parameters: &Parameters, data: InitialSyncData) - where - C: Configuration, - { - let InitialSyncData { - utxo_data, - membership_proof_data, - nullifier_count, - } = data; - let sorted_utxo_data = C::UtxoAccumulator::sort_items( - utxo_data - .iter() - .map(|utxo| functions::item_hash::(parameters, utxo)) - .collect(), - ); - for (old_vector, new_vector) in self.utxo_data.iter_mut().zip(sorted_utxo_data.into_iter()) + pub fn extend(&mut self, new_request: InitialSyncRequest) { + for (old_vector, new_vector) in self + .utxo_data + .iter_mut() + .zip(new_request.utxo_data.into_iter()) { old_vector.extend(new_vector) } - self.membership_proof_data = membership_proof_data; - self.nullifier_count = nullifier_count; + self.membership_proof_data = new_request.membership_proof_data; + self.nullifier_count = new_request.nullifier_count; + } + + /// Extends `self` with `parameters` and `data`. + #[inline] + pub fn extend_with_data(&mut self, parameters: &Parameters, data: InitialSyncData) { + let new_request = Self::from_initial_sync_data(parameters, data); + self.extend(new_request); } } @@ -422,6 +421,24 @@ where self.data .prune(parameters, &self.origin_checkpoint, checkpoint) } + + /// + #[inline] + pub fn utxo_count(&self, parameters: &Parameters) -> Vec + where + C: Configuration, + { + C::UtxoAccumulator::sort_items( + self.data + .utxo_note_data + .iter() + .map(|(utxo, _)| functions::item_hash::(parameters, utxo)) + .collect(), + ) + .into_iter() + .map(|utxos| utxos.len()) + .collect() + } } /// Signer Synchronization Response @@ -1463,6 +1480,24 @@ where pub fn transfer_parameters(&self) -> &Parameters { &self.parameters.parameters } + + /// + #[inline] + pub fn sbt_sync( + &mut self, + request: SyncRequest, + ) -> Result, SyncError> { + functions::sbt_sync( + &self.parameters, + self.state + .authorization_context + .as_mut() + .ok_or(SyncError::MissingProofAuthorizationKey)?, + &mut self.state.assets, + &mut self.state.checkpoint, + request, + ) + } } impl Connection for Signer @@ -1481,6 +1516,14 @@ where Box::pin(async move { Ok(self.sync(request)) }) } + #[inline] + fn sbt_sync( + &mut self, + request: SyncRequest, + ) -> LocalBoxFutureResult, Self::Error> { + Box::pin(async move { Ok(self.sbt_sync(request)) }) + } + #[inline] fn initial_sync( &mut self, diff --git a/manta-pay/src/signer/base.rs b/manta-pay/src/signer/base.rs index bcba9b7e9..258588f60 100644 --- a/manta-pay/src/signer/base.rs +++ b/manta-pay/src/signer/base.rs @@ -114,12 +114,18 @@ impl signer::Checkpoint for Checkpoint { #[inline] fn update_from_utxo_count(&mut self, utxo_count: Vec) { - self.receiver_index = manta_util::Array(utxo_count.try_into().unwrap_or_else(|_| { - panic!( - "Utxo count must have {} elements", - MerkleTreeConfiguration::FOREST_WIDTH - ) - })) + assert_eq!( + utxo_count.len(), + MerkleTreeConfiguration::FOREST_WIDTH, + "Utxo count must have {} elements", + MerkleTreeConfiguration::FOREST_WIDTH + ); + self.receiver_index = self + .receiver_index + .into_iter() + .zip(utxo_count.into_iter()) + .map(|(a, b)| a + b) + .collect(); } /// Prunes the `data` by comparing `origin` and `signer_checkpoint` and checks if updating the diff --git a/manta-pay/src/signer/client/http.rs b/manta-pay/src/signer/client/http.rs index fa5d0668c..8e75978e2 100644 --- a/manta-pay/src/signer/client/http.rs +++ b/manta-pay/src/signer/client/http.rs @@ -102,6 +102,14 @@ impl signer::Connection for Client { Box::pin(self.post_request("sync", request)) } + #[inline] + fn sbt_sync( + &mut self, + request: SyncRequest, + ) -> LocalBoxFutureResult, Self::Error> { + Box::pin(self.post_request("sbt_sync", request)) + } + #[inline] fn initial_sync( &mut self, diff --git a/manta-pay/src/signer/client/websocket.rs b/manta-pay/src/signer/client/websocket.rs index 6c7811f59..7e5f3bb84 100644 --- a/manta-pay/src/signer/client/websocket.rs +++ b/manta-pay/src/signer/client/websocket.rs @@ -140,6 +140,14 @@ impl signer::Connection for Client { Box::pin(self.send("sync", request)) } + #[inline] + fn sbt_sync( + &mut self, + request: SyncRequest, + ) -> LocalBoxFutureResult, Self::Error> { + Box::pin(self.send("sbt_sync", request)) + } + #[inline] fn initial_sync( &mut self, From c4f870468e8384bc54f74745c5fcf809f158502b Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 5 Apr 2023 14:51:15 +0200 Subject: [PATCH 56/58] sbt sync wallet --- manta-accounting/src/wallet/mod.rs | 73 ++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index a1bd650c2..2d85f5d6f 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -272,6 +272,16 @@ where Ok(()) } + /// + #[inline] + pub async fn sbt_sync(&mut self) -> Result<(), Error> + where + L: ledger::Read, Checkpoint = S::Checkpoint>, + { + while self.sbt_sync_partial().await?.is_continue() {} + Ok(()) + } + /// Pulls data from the ledger, synchronizing the wallet and balance state. This method /// builds a [`InitialSyncRequest`] by continuously calling [`read`](ledger::Read::read) /// until all the ledger data has arrived. Once the request is built, it executes @@ -298,7 +308,7 @@ where S::Checkpoint: signer::Checkpoint, { let mut is_continue = true; - let mut checkpoint = Default::default(); + let mut checkpoint = S::Checkpoint::default(); let mut request = InitialSyncRequest::::default(); let parameters = self .signer @@ -309,15 +319,12 @@ where let ReadResponse { should_continue, data, - } = self - .ledger - .read(&checkpoint) - .await - .map_err(Error::LedgerConnectionError)?; + } = self.read_from_ledger().await?; + let more = InitialSyncRequest::from_initial_sync_data(¶meters, data); is_continue = should_continue; - request.extend_with_data(¶meters, data); checkpoint - .update_from_utxo_count(request.utxo_data.iter().map(|utxos| utxos.len()).collect()) + .update_from_utxo_count(more.utxo_data.iter().map(|utxos| utxos.len()).collect()); + request.extend(more); } self.signer_initial_sync(request).await?; Ok(()) @@ -341,6 +348,36 @@ where self.sync_with().await } + /// + #[inline] + async fn read_from_ledger(&mut self) -> Result, Error> + where + L: ledger::Read, + { + self.ledger + .read(&self.checkpoint) + .await + .map_err(Error::LedgerConnectionError) + } + + /// + #[inline] + pub async fn sbt_sync_partial(&mut self) -> Result> + where + L: ledger::Read, Checkpoint = S::Checkpoint>, + { + let ReadResponse { + should_continue, + data, + } = self.read_from_ledger().await?; + self.signer_sbt_sync(SyncRequest { + origin_checkpoint: self.checkpoint.clone(), + data, + }) + .await?; + Ok(ControlFlow::should_continue(should_continue)) + } + /// Pulls data from the ledger, synchronizing the wallet and balance state. #[inline] async fn sync_with(&mut self) -> Result> @@ -350,11 +387,7 @@ where let ReadResponse { should_continue, data, - } = self - .ledger - .read(&self.checkpoint) - .await - .map_err(Error::LedgerConnectionError)?; + } = self.read_from_ledger().await?; self.signer_sync(SyncRequest { origin_checkpoint: self.checkpoint.clone(), data, @@ -431,6 +464,20 @@ where self.process_sync_response(response) } + /// + #[inline] + async fn signer_sbt_sync( + &mut self, + request: SyncRequest, + ) -> Result<(), Error> { + let response = self + .signer + .sbt_sync(request) + .await + .map_err(Error::SignerConnectionError)?; + self.process_sync_response(response) + } + /// Checks if `transaction` can be executed on the balance state of `self`, returning the /// kind of update that should be performed on the balance state if the transaction is /// successfully posted to the ledger. From f2fd6fe8a574fe3658635faa2c5f006a13d62f30 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 5 Apr 2023 16:10:21 +0200 Subject: [PATCH 57/58] docs --- manta-accounting/src/wallet/mod.rs | 44 ++++++++++++++++++- .../src/wallet/signer/functions.rs | 2 +- manta-accounting/src/wallet/signer/mod.rs | 33 ++++++++++---- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/manta-accounting/src/wallet/mod.rs b/manta-accounting/src/wallet/mod.rs index 2d85f5d6f..dce1a2f36 100644 --- a/manta-accounting/src/wallet/mod.rs +++ b/manta-accounting/src/wallet/mod.rs @@ -272,7 +272,27 @@ where Ok(()) } + /// Pulls data from the ledger, synchronizing the wallet and balance state. This method loops + /// continuously calling [`sbt_sync_partial`](Self::sbt_sync_partial) until all the ledger data has + /// arrived at and has been synchronized with the wallet. + /// + /// # Failure Conditions /// + /// This method returns an element of type [`Error`] on failure, which can result from any + /// number of synchronization issues between the wallet, the ledger, and the signer. See the + /// [`InconsistencyError`] type for more information on the kinds of errors that can occur and + /// how to resolve them. + /// + /// # Note + /// + /// In general, this method does not update the [`Utxo`] accumulator, thus making the new assets + /// effectively non-spendable. Therefore, this method should only be used when the pallet does not + /// allow [`PrivateTransfer`]s or [`ToPublic`] transactions, for example in the case of + /// Soul-Bound Tokens (SBTs). + /// + /// [`Utxo`]: Configuration::Utxo + /// [`PrivateTransfer`]: crate::transfer::canonical::PrivateTransfer + /// [`ToPublic`]: crate::transfer::canonical::ToPublic #[inline] pub async fn sbt_sync(&mut self) -> Result<(), Error> where @@ -348,7 +368,7 @@ where self.sync_with().await } - /// + /// Reads data from the ledger. #[inline] async fn read_from_ledger(&mut self) -> Result, Error> where @@ -360,7 +380,27 @@ where .map_err(Error::LedgerConnectionError) } + /// Pulls data from the ledger, synchronizing the wallet and balance state. This method returns + /// a [`ControlFlow`] for matching against to determine if the wallet requires more + /// synchronization. + /// + /// # Failure Conditions + /// + /// This method returns an element of type [`Error`] on failure, which can result from any + /// number of synchronization issues between the wallet, the ledger, and the signer. See the + /// [`InconsistencyError`] type for more information on the kinds of errors that can occur and + /// how to resolve them. /// + /// # Note + /// + /// In general, this method does not update the [`Utxo`] accumulator, thus making the new assets + /// effectively non-spendable. Therefore, this method should only be used when the pallet does not + /// allow [`PrivateTransfer`]s or [`ToPublic`] transactions, for example in the case of + /// Soul-Bound Tokens (SBTs). + /// + /// [`Utxo`]: Configuration::Utxo + /// [`PrivateTransfer`]: crate::transfer::canonical::PrivateTransfer + /// [`ToPublic`]: crate::transfer::canonical::ToPublic #[inline] pub async fn sbt_sync_partial(&mut self) -> Result> where @@ -464,7 +504,7 @@ where self.process_sync_response(response) } - /// + /// Performs an sbt synchronization with the signer against the given `request`. #[inline] async fn signer_sbt_sync( &mut self, diff --git a/manta-accounting/src/wallet/signer/functions.rs b/manta-accounting/src/wallet/signer/functions.rs index a0be64779..a91608612 100644 --- a/manta-accounting/src/wallet/signer/functions.rs +++ b/manta-accounting/src/wallet/signer/functions.rs @@ -691,7 +691,7 @@ where } } -/// +/// Updates `assets` and `checkpoint`, returning the new asset distribution. #[inline] pub fn sbt_sync( parameters: &SignerParameters, diff --git a/manta-accounting/src/wallet/signer/mod.rs b/manta-accounting/src/wallet/signer/mod.rs index 76605370c..2a0729d66 100644 --- a/manta-accounting/src/wallet/signer/mod.rs +++ b/manta-accounting/src/wallet/signer/mod.rs @@ -79,7 +79,19 @@ where request: SyncRequest, ) -> LocalBoxFutureResult, Self::Error>; + /// Pushes updates from the ledger to the wallet, synchronizing it with the ledger state and + /// returning an updated asset distribution. + /// + /// # Implementation Note + /// + /// Implementations of this method must return the same updated asset distribution and the same + /// balance update as [`sync`](Connection::sync). However, they do not have to update the [`Utxo`] + /// accumulator, thus making the new assets effectively non-spendable. Therefore, this method should + /// only be used when the pallet does not allow [`PrivateTransfer`]s or [`ToPublic`] transactions, + /// for example in the case of a pallet for Soul-Bound Tokens (SBTs). /// + /// [`PrivateTransfer`]: transfer::canonical::PrivateTransfer + /// [`ToPublic`]: transfer::canonical::ToPublic fn sbt_sync( &mut self, request: SyncRequest, @@ -258,18 +270,15 @@ where } } - /// + /// Extends `self` with `request`. #[inline] - pub fn extend(&mut self, new_request: InitialSyncRequest) { - for (old_vector, new_vector) in self - .utxo_data - .iter_mut() - .zip(new_request.utxo_data.into_iter()) + pub fn extend(&mut self, request: InitialSyncRequest) { + for (old_vector, new_vector) in self.utxo_data.iter_mut().zip(request.utxo_data.into_iter()) { old_vector.extend(new_vector) } - self.membership_proof_data = new_request.membership_proof_data; - self.nullifier_count = new_request.nullifier_count; + self.membership_proof_data = request.membership_proof_data; + self.nullifier_count = request.nullifier_count; } /// Extends `self` with `parameters` and `data`. @@ -422,7 +431,7 @@ where .prune(parameters, &self.origin_checkpoint, checkpoint) } - /// + /// Returns the [`Utxo`] count of `self`. #[inline] pub fn utxo_count(&self, parameters: &Parameters) -> Vec where @@ -1481,7 +1490,13 @@ where &self.parameters.parameters } + /// Updates the internal ledger state, returning the new asset distribution. + /// + /// # Note /// + /// This method updates the checkpoint and assetmap, but it does not update + /// the [`UtxoAccumulator`](Configuration::UtxoAccumulator). Therefore, it should + /// only be used for non-spendable assets such as SBTs. #[inline] pub fn sbt_sync( &mut self, From b803418dca57697618d21cd087e7b5c06d7104a3 Mon Sep 17 00:00:00 2001 From: SupremoUGH Date: Wed, 5 Apr 2023 16:11:42 +0200 Subject: [PATCH 58/58] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f1faaa6e..da374db74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added +- [\#335](https://github.com/Manta-Network/manta-rs/pull/335) SBT synchronization method for the signer. - [\#330](https://github.com/Manta-Network/manta-rs/pull/330) Merkle tree batch insertions. ### Changed